Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
Lint code (#25)
kkkwjxa30cb2e7b9bd
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.ymlindex 4e87bee00..e2d00d182 100644--- a/.github/workflows/lint.yml+++ b/.github/workflows/lint.yml@@ -50,11 +50,8 @@ jobs:- name: Install dependenciesrun: uv sync- - name: Check code formatting with black- run: uv run black --check --diff openviking/-- - name: Check import sorting with isort- run: uv run isort --check-only --diff openviking/+ - name: Format with ruff+ run: uv run ruff format --check openviking/- name: Lint with ruffrun: uv run ruff check openviking/diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlnew file mode 100644index 000000000..e08c5c4dd--- /dev/null+++ b/.pre-commit-config.yaml@@ -0,0 +1,7 @@+repos:+ - repo: https://github.com/astral-sh/ruff-pre-commit+ rev: v0.14.14+ hooks:+ - id: ruff+ args: [ --fix ]+ - id: ruff-formatdiff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex b75fdbaa1..3b69757e8 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -140,17 +140,30 @@ We use the following tools to maintain code consistency:| Tool | Purpose | Config ||------|---------|--------|-| **Black** | Code formatting | `pyproject.toml` |-| **isort** | Import sorting | `pyproject.toml` |-| **Ruff** | Linting | `pyproject.toml` |+| **Ruff** | Linting, Formatting, Import sorting | `pyproject.toml` || **mypy** | Type checking | `pyproject.toml` |+### Automated Checks (Recommended)++We use [pre-commit](https://pre-commit.com/) to automatically run these checks before every commit. This ensures your code always meets the standards without manual effort.++1. **Install pre-commit**:+ ```bash+ pip install pre-commit+ ```++2. **Install the git hooks**:+ ```bash+ pre-commit install+ ```++Now, `ruff` (check & format) will run automatically when you run `git commit`. If any check fails, it may automatically fix the file. You just need to add the changes and commit again.+### Running Checks```bash# Format code-black openviking/-isort openviking/+ruff format openviking/# Lintruff check openviking/diff --git a/examples/query/add.py b/examples/query/add.pyindex b4e0e9f54..bafe37249 100644--- a/examples/query/add.py+++ b/examples/query/add.py@@ -2,12 +2,12 @@"""Add Resource - CLI tool to add documents to OpenViking database"""-import boring_logging_config # Configure logging (set OV_DEBUG=1 for debug mode)+import argparseimport jsonimport sys-import osfrom pathlib import Path+import openviking as ovfrom openviking.utils.config.open_viking_config import OpenVikingConfig@@ -23,7 +23,7 @@ def add_resource(resource_path: str, config_path: str = "./ov.conf", data_path:"""# Load configprint(f"π Loading config from: {config_path}")- with open(config_path, 'r') as f:+ with open(config_path, "r") as f:config_dict = json.load(f)config = OpenVikingConfig.from_dict(config_dict)@@ -37,7 +37,7 @@ def add_resource(resource_path: str, config_path: str = "./ov.conf", data_path:print(f"π Adding resource: {resource_path}")# Check if it's a file and exists- if not resource_path.startswith('http'):+ if not resource_path.startswith("http"):path = Path(resource_path).expanduser()if not path.exists():print(f"β Error: File not found: {path}")@@ -46,8 +46,8 @@ def add_resource(resource_path: str, config_path: str = "./ov.conf", data_path:result = client.add_resource(path=resource_path)# Check result- if result and 'root_uri' in result:- root_uri = result['root_uri']+ if result and "root_uri" in result:+ root_uri = result["root_uri"]print(f"β Resource added: {root_uri}\n")# Wait for processing@@ -58,10 +58,10 @@ def add_resource(resource_path: str, config_path: str = "./ov.conf", data_path:print("π Resource is now searchable in the database!")return True- elif result and result.get('status') == 'error':- print(f"\nβ οΈ Resource had parsing issues:")- if 'errors' in result:- for error in result['errors'][:3]:+ elif result and result.get("status") == "error":+ print("\nβ οΈ Resource had parsing issues:")+ if "errors" in result:+ for error in result["errors"][:3]:print(f" - {error}")print("\nπ‘ Some content may still be searchable.")return False@@ -73,6 +73,7 @@ def add_resource(resource_path: str, config_path: str = "./ov.conf", data_path:except Exception as e:print(f"\nβ Error: {e}")import traceback+traceback.print_exc()return False@@ -107,33 +108,29 @@ Notes:- URLs are automatically downloaded and processed- Large files may take several minutes to process- The resource becomes searchable after processing completes- """+ """,)parser.add_argument(- 'resource',- type=str,- help='Path to file/directory or URL to add to the database'+ "resource", type=str, help="Path to file/directory or URL to add to the database")parser.add_argument(- '--config',- type=str,- default='./ov.conf',- help='Path to config file (default: ./ov.conf)'+ "--config", type=str, default="./ov.conf", help="Path to config file (default: ./ov.conf)")parser.add_argument(- '--data',- type=str,- default='./data',- help='Path to data directory (default: ./data)'+ "--data", type=str, default="./data", help="Path to data directory (default: ./data)")args = parser.parse_args()# Expand user paths- resource_path = str(Path(args.resource).expanduser()) if not args.resource.startswith('http') else args.resource+ resource_path = (+ str(Path(args.resource).expanduser())+ if not args.resource.startswith("http")+ else args.resource+ )# Add the resourcesuccess = add_resource(resource_path, args.config, args.data)diff --git a/examples/query/boring_logging_config.py b/examples/query/boring_logging_config.pyindex 531781b4e..8d7cacc8c 100644--- a/examples/query/boring_logging_config.py+++ b/examples/query/boring_logging_config.py@@ -2,63 +2,102 @@Centralized logging configurationSet OV_DEBUG=1 environment variable to enable debug logging"""+import loggingimport logging.configimport osimport warnings# Suppress warnings-warnings.filterwarnings('ignore')+warnings.filterwarnings("ignore")# Check debug mode from environment-DEBUG = os.environ.get('OV_DEBUG') == '1'+DEBUG = os.environ.get("OV_DEBUG") == "1"if DEBUG:# Debug mode - show all logslogging.basicConfig(- level=logging.INFO,- format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'+ level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")else:# Production mode - aggressively suppress all logs- logging.config.dictConfig({- 'version': 1,- 'disable_existing_loggers': True,- 'formatters': {- 'null': {- 'format': ''- }- },- 'handlers': {- 'null': {- 'class': 'logging.NullHandler',+ logging.config.dictConfig(+ {+ "version": 1,+ "disable_existing_loggers": True,+ "formatters": {"null": {"format": ""}},+ "handlers": {+ "null": {+ "class": "logging.NullHandler",+ },+ },+ "root": {"level": "CRITICAL", "handlers": ["null"]},+ "loggers": {+ # Suppress all OpenViking loggers+ "openviking": {"level": "CRITICAL", "handlers": ["null"], "propagate": False},+ "openviking.agfs_manager": {+ "level": "CRITICAL",+ "handlers": ["null"],+ "propagate": False,+ },+ "openviking.storage": {+ "level": "CRITICAL",+ "handlers": ["null"],+ "propagate": False,+ },+ "openviking.storage.viking_vector_index_backend": {+ "level": "CRITICAL",+ "handlers": ["null"],+ "propagate": False,+ },+ "openviking.storage.queuefs": {+ "level": "CRITICAL",+ "handlers": ["null"],+ "propagate": False,+ },+ "openviking.storage.queuefs.queue_manager": {+ "level": "CRITICAL",+ "handlers": ["null"],+ "propagate": False,+ },+ "openviking.storage.vikingdb_manager": {+ "level": "CRITICAL",+ "handlers": ["null"],+ "propagate": False,+ },+ "openviking.storage.viking_fs": {+ "level": "CRITICAL",+ "handlers": ["null"],+ "propagate": False,+ },+ "openviking.async_client": {+ "level": "CRITICAL",+ "handlers": ["null"],+ "propagate": False,+ },+ "openviking.parse": {"level": "CRITICAL", "handlers": ["null"], "propagate": False},+ "openviking.parse.parsers": {+ "level": "CRITICAL",+ "handlers": ["null"],+ "propagate": False,+ },+ "openviking.parse.parsers.markdown": {+ "level": "CRITICAL",+ "handlers": ["null"],+ "propagate": False,+ },+ "openviking.storage.queuefs.semantic_processor": {+ "level": "CRITICAL",+ "handlers": ["null"],+ "propagate": False,+ },+ "apscheduler": {"level": "CRITICAL", "handlers": ["null"], "propagate": False},},- },- 'root': {- 'level': 'CRITICAL',- 'handlers': ['null']- },- 'loggers': {- # Suppress all OpenViking loggers- 'openviking': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},- 'openviking.agfs_manager': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},- 'openviking.storage': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},- 'openviking.storage.viking_vector_index_backend': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},- 'openviking.storage.queuefs': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},- 'openviking.storage.queuefs.queue_manager': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},- 'openviking.storage.vikingdb_manager': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},- 'openviking.storage.viking_fs': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},- 'openviking.async_client': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},- 'openviking.parse': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},- 'openviking.parse.parsers': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},- 'openviking.parse.parsers.markdown': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},- 'openviking.storage.queuefs.semantic_processor': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},- 'apscheduler': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},}- })+ )# Additional enforcement: set all loggers after config- for logger_name in ['openviking', 'apscheduler']:+ for logger_name in ["openviking", "apscheduler"]:logger = logging.getLogger(logger_name)logger.setLevel(logging.CRITICAL)logger.propagate = Falsediff --git a/examples/query/query.py b/examples/query/query.pyindex 92875dbcd..4e98ff616 100644--- a/examples/query/query.py+++ b/examples/query/query.py@@ -2,13 +2,11 @@"""Query - CLI tool to search and generate answers using OpenViking + LLM"""+import argparseimport sys-import textwrapimport threading-import boring_logging_config # Configure logging (set OV_DEBUG=1 for debug mode)-from recipe import Recipefrom rich import boxfrom rich.console import Console@@ -79,7 +77,7 @@ def query(temperature: float = 0.7,max_tokens: int = 2048,score_threshold: float = 0.2,- verbose: bool = False+ verbose: bool = False,):"""Query the database and generate an answer using LLM@@ -111,13 +109,23 @@ def query(try:pipeline = Recipe(config_path=config_path, data_path=data_path)except Exception as e:- console.print(Panel(f"β Error initializing pipeline: {e}", style="bold red", padding=(0, 1)))+ console.print(+ Panel(f"β Error initializing pipeline: {e}", style="bold red", padding=(0, 1))+ )return Falsetry:# Display question with constrained widthquestion_text = Text(question, style="bold yellow")- console.print(Panel(question_text, title="β Roger That", style="bold", padding=(0, 1), width=PANEL_WIDTH))+ console.print(+ Panel(+ question_text,+ title="β Roger That",+ style="bold",+ padding=(0, 1),+ width=PANEL_WIDTH,+ )+ )console.print()# Query with loading spinner@@ -128,20 +136,28 @@ def query(search_top_k=top_k,temperature=temperature,max_tokens=max_tokens,- score_threshold=score_threshold+ score_threshold=score_threshold,+ )+ answer_text = Text(result["answer"], style="white")+ console.print(+ Panel(+ answer_text,+ title="π Check This Out",+ style="bold bright_cyan",+ padding=(1, 1),+ width=PANEL_WIDTH,+ ))- answer_text = Text(result['answer'], style="white")- console.print(Panel(answer_text, title="π Check This Out", style="bold bright_cyan", padding=(1, 1), width=PANEL_WIDTH))console.print()# Show sources- if result['context']:+ if result["context"]:sources_table = Table(title=f"π Sources ({len(result['context'])} documents)",box=box.ROUNDED,show_header=True,header_style="bold magenta",- title_style="bold magenta"+ title_style="bold magenta",)sources_table.add_column("#", style="cyan", width=4)sources_table.add_column("File", style="bold white")@@ -151,26 +167,38 @@ def query(sources_table.add_column("URI", style="dim")sources_table.add_column("Preview", style="dim")- for i, ctx in enumerate(result['context'], 1):- uri_parts = ctx['uri'].split('/')- filename = uri_parts[-1] if uri_parts else ctx['uri']+ for i, ctx in enumerate(result["context"], 1):+ uri_parts = ctx["uri"].split("/")+ filename = uri_parts[-1] if uri_parts else ctx["uri"]score_text = Text(f"{ctx['score']:.4f}", style="bold green")if verbose:- preview = ctx['content'][:100] + "..." if len(ctx['content']) > 100 else ctx['content']- sources_table.add_row(str(i), filename, score_text, ctx['uri'], preview)+ preview = (+ ctx["content"][:100] + "..."+ if len(ctx["content"]) > 100+ else ctx["content"]+ )+ sources_table.add_row(str(i), filename, score_text, ctx["uri"], preview)else:sources_table.add_row(str(i), filename, score_text)console.print(sources_table)else:- console.print(Panel("β οΈ No relevant sources found", style="yellow", padding=(0, 1), width=PANEL_WIDTH))+ console.print(+ Panel(+ "β οΈ No relevant sources found",+ style="yellow",+ padding=(0, 1),+ width=PANEL_WIDTH,+ )+ )return Trueexcept Exception as e:console.print(Panel(f"β Error during query: {e}", style="bold red", padding=(0, 1)))import traceback+traceback.print_exc()return False@@ -216,68 +244,54 @@ Score Threshold Guide:0.0-0.1 β Very permissive, includes low relevance results0.2 β Balanced (default)0.3-0.5 β Strict, only highly relevant results- """+ """,)- parser.add_argument(- 'question',- type=str,- help='Your question to ask the LLM'- )+ parser.add_argument("question", type=str, help="Your question to ask the LLM")parser.add_argument(- '--config',- type=str,- default='./ov.conf',- help='Path to config file (default: ./ov.conf)'+ "--config", type=str, default="./ov.conf", help="Path to config file (default: ./ov.conf)")parser.add_argument(- '--data',- type=str,- default='./data',- help='Path to data directory (default: ./data)'+ "--data", type=str, default="./data", help="Path to data directory (default: ./data)")parser.add_argument(- '--top-k',+ "--top-k",type=int,default=5,- help='Number of search results to use as context (default: 5)'+ help="Number of search results to use as context (default: 5)",)parser.add_argument(- '--temperature',- type=float,- default=0.7,- help='LLM temperature 0.0-1.0 (default: 0.7)'+ "--temperature", type=float, default=0.7, help="LLM temperature 0.0-1.0 (default: 0.7)")parser.add_argument(- '--max-tokens',- type=int,- default=2048,- help='Maximum tokens to generate (default: 2048)'+ "--max-tokens", type=int, default=2048, help="Maximum tokens to generate (default: 2048)")parser.add_argument(- '--score-threshold',+ "--score-threshold",type=float,default=0.2,- help='Minimum relevance score for search results (default: 0.2)'+ help="Minimum relevance score for search results (default: 0.2)",)- parser.add_argument(- '--verbose', '-v',- action='store_true',- help='Show detailed information'- )+ parser.add_argument("--verbose", "-v", action="store_true", help="Show detailed information")args = parser.parse_args()# Validate temperatureif not 0.0 <= args.temperature <= 1.0:- console.print(Panel("β Error: Temperature must be between 0.0 and 1.0", style="bold red", padding=(0, 1)))+ console.print(+ Panel(+ "β Error: Temperature must be between 0.0 and 1.0",+ style="bold red",+ padding=(0, 1),+ )+ )sys.exit(1)# Validate top-k@@ -286,7 +300,13 @@ Score Threshold Guide:sys.exit(1)if not 0.0 <= args.score_threshold <= 1.0:- console.print(Panel("β Error: score-threshold must be between 0.0 and 1.0", style="bold red", padding=(0, 1)))+ console.print(+ Panel(+ "β Error: score-threshold must be between 0.0 and 1.0",+ style="bold red",+ padding=(0, 1),+ )+ )sys.exit(1)# Run query@@ -298,7 +318,7 @@ Score Threshold Guide:temperature=args.temperature,max_tokens=args.max_tokens,score_threshold=args.score_threshold,- verbose=args.verbose+ verbose=args.verbose,)sys.exit(0 if success else 1)diff --git a/examples/query/recipe.py b/examples/query/recipe.pyindex e2631922c..2441cf1ea 100644--- a/examples/query/recipe.py+++ b/examples/query/recipe.py@@ -3,12 +3,14 @@RAG Pipeline - Retrieval-Augmented Generation using OpenViking + LLMFocused on querying and answer generation, not resource management"""-import boring_logging_config # Configure logging (set OV_DEBUG=1 for debug mode)-import openviking as ov+import json+from typing import Any, Dict, List, Optional+import requests++import openviking as ovfrom openviking.utils.config.open_viking_config import OpenVikingConfig-from typing import Optional, List, Dict, Anyclass Recipe:@@ -30,21 +32,27 @@ class Recipe:data_path: Path to OpenViking data directory"""# Load configuration- with open(config_path, 'r') as f:+ with open(config_path, "r") as f:self.config_dict = json.load(f)# Extract LLM config- self.vlm_config = self.config_dict.get('vlm', {})- self.api_base = self.vlm_config.get('api_base')- self.api_key = self.vlm_config.get('api_key')- self.model = self.vlm_config.get('model')+ self.vlm_config = self.config_dict.get("vlm", {})+ self.api_base = self.vlm_config.get("api_base")+ self.api_key = self.vlm_config.get("api_key")+ self.model = self.vlm_config.get("model")# Initialize OpenViking clientconfig = OpenVikingConfig.from_dict(self.config_dict)self.client = ov.SyncOpenViking(path=data_path, config=config)self.client.initialize()- def search(self, query: str, top_k: int = 3, target_uri: Optional[str] = None, score_threshold: float = 0.2) -> List[Dict[str, Any]]:+ def search(+ self,+ query: str,+ top_k: int = 3,+ target_uri: Optional[str] = None,+ score_threshold: float = 0.2,+ ) -> List[Dict[str, Any]]:"""Search for relevant content using semantic search@@ -69,24 +77,30 @@ class Recipe:try:# Try to read the resourcecontent = self.client.read(resource.uri)- search_results.append({- 'uri': resource.uri,- 'score': resource.score,- 'content': content[:1000] # Limit content length- })- print(f" {i+1}. {resource.uri} (score: {resource.score:.4f})")+ search_results.append(+ {+ "uri": resource.uri,+ "score": resource.score,+ "content": content[:1000], # Limit content length+ }+ )+ print(f" {i + 1}. {resource.uri} (score: {resource.score:.4f})")except Exception as e:# Handle directories - read their abstract insteadif "is a directory" in str(e):try:abstract = self.client.abstract(resource.uri)- search_results.append({- 'uri': resource.uri,- 'score': resource.score,- 'content': f"[Directory Abstract] {abstract[:1000]}"- })- print(f" {i+1}. {resource.uri} (score: {resource.score:.4f}) [directory]")+ search_results.append(+ {+ "uri": resource.uri,+ "score": resource.score,+ "content": f"[Directory Abstract] {abstract[:1000]}",+ }+ )+ print(+ f" {i + 1}. {resource.uri} (score: {resource.score:.4f}) [directory]"+ )except:# Skip if we can't get abstractcontinue@@ -110,21 +124,13 @@ class Recipe:"""url = f"{self.api_base}/chat/completions"- headers = {- "Content-Type": "application/json",- "Authorization": f"Bearer {self.api_key}"- }+ headers = {"Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}"}payload = {"model": self.model,- "messages": [- {- "role": "user",- "content": prompt- }- ],+ "messages": [{"role": "user", "content": prompt}],"temperature": temperature,- "max_tokens": max_tokens+ "max_tokens": max_tokens,}print(f"π€ Calling LLM: {self.model}")@@ -132,17 +138,19 @@ class Recipe:response.raise_for_status()result = response.json()- answer = result['choices'][0]['message']['content']+ answer = result["choices"][0]["message"]["content"]return answer- def query(self,- user_query: str,- search_top_k: int = 3,- temperature: float = 0.7,- max_tokens: int = 2048,- system_prompt: Optional[str] = None,- score_threshold: float = 0.2) -> Dict[str, Any]:+ def query(+ self,+ user_query: str,+ search_top_k: int = 3,+ temperature: float = 0.7,+ max_tokens: int = 2048,+ system_prompt: Optional[str] = None,+ score_threshold: float = 0.2,+ ) -> Dict[str, Any]:"""Full RAG pipeline: search β retrieve β generate@@ -158,20 +166,24 @@ class Recipe:Dictionary with answer, context, and metadata"""# Step 1: Search for relevant content- search_results = self.search(user_query, top_k=search_top_k, score_threshold=score_threshold)+ search_results = self.search(+ user_query, top_k=search_top_k, score_threshold=score_threshold+ )if not search_results:return {- 'answer': "I couldn't find any relevant information to answer your question.",- 'context': [],- 'query': user_query+ "answer": "I couldn't find any relevant information to answer your question.",+ "context": [],+ "query": user_query,}# Step 2: Build context from search results- context_text = "\n\n".join([- f"[Source {i+1}] (relevance: {r['score']:.4f})\n{r['content']}"- for i, r in enumerate(search_results)- ])+ context_text = "\n\n".join(+ [+ f"[Source {i + 1}] (relevance: {r['score']:.4f})\n{r['content']}"+ for i, r in enumerate(search_results)+ ]+ )# Step 3: Build the promptif system_prompt:@@ -188,12 +200,7 @@ class Recipe:answer = self.call_llm(prompt, temperature=temperature, max_tokens=max_tokens)# Return full result- return {- 'answer': answer,- 'context': search_results,- 'query': user_query,- 'prompt': prompt- }+ return {"answer": answer, "context": search_results, "query": user_query, "prompt": prompt}def close(self):"""Clean up resources"""diff --git a/examples/quick_start.py b/examples/quick_start.pyindex 4dd5ff45f..2e154d430 100644--- a/examples/quick_start.py+++ b/examples/quick_start.py@@ -6,24 +6,26 @@ try:client.initialize()# Add resource (URL, file, or directory)- res = client.add_resource(path="https://raw.githubusercontent.com/volcengine/MineContext/refs/heads/main/README_zh.md")- root_uri = res['root_uri']- res = client.ls(root_uri) # Explore resource tree+ res = client.add_resource(+ path="https://raw.githubusercontent.com/volcengine/MineContext/refs/heads/main/README_zh.md"+ )+ root_uri = res["root_uri"]+ res = client.ls(root_uri) # Explore resource treeprint(f"Directory structure:\n{res}\n")- res = client.glob(pattern="**/*.md", uri=root_uri) # use glob to find markdown files- if res['matches']:- content = client.read(res['matches'][0])+ res = client.glob(pattern="**/*.md", uri=root_uri) # use glob to find markdown files+ if res["matches"]:+ content = client.read(res["matches"][0])print(f"Content preview: {content[:200]}...\n")print(" Wait for semantic processing...")client.wait_processed()- abstract = client.abstract(root_uri) # Get abstract- overview = client.overview(root_uri) # Get overview+ abstract = client.abstract(root_uri) # Get abstract+ overview = client.overview(root_uri) # Get overviewprint(f"Abstract:\n{abstract}\n\nOverview:\n{overview}\n")-- results = client.find("what is openviking", target_uri=root_uri) # Semantic search++ results = client.find("what is openviking", target_uri=root_uri) # Semantic searchprint("Search results:")for r in results.resources:print(f" {r.uri} (score: {r.score:.4f})")@@ -31,4 +33,4 @@ try:client.close()except Exception as e:- print(f"Error: {e}")\ No newline at end of file+ print(f"Error: {e}")diff --git a/openviking/__init__.py b/openviking/__init__.pyindex 21557d911..3fc9536e8 100644--- a/openviking/__init__.py+++ b/openviking/__init__.py@@ -6,7 +6,7 @@ OpenViking - An Agent-native context databaseData in, Context out."""-from openviking.client import SyncOpenViking, AsyncOpenViking+from openviking.client import AsyncOpenViking, SyncOpenVikingfrom openviking.session import SessionOpenViking = SyncOpenVikingdiff --git a/openviking/agfs_manager.py b/openviking/agfs_manager.pyindex 1e05d1155..6898e5bd3 100644--- a/openviking/agfs_manager.py+++ b/openviking/agfs_manager.py@@ -8,12 +8,15 @@ import socketimport subprocessimport timefrom pathlib import Path-from typing import Optional+from typing import TYPE_CHECKING, Optionalimport yamlfrom openviking.utils import get_logger+if TYPE_CHECKING:+ from openviking.utils.config.agfs_config import AGFSConfig+logger = get_logger(__name__)@@ -80,7 +83,6 @@ class AGFSManager:atexit.register(self.stop)-@propertydef vikingfs_path(self) -> Path:"""AGFS LocalFS data directory."""@@ -211,7 +213,7 @@ class AGFSManager:try:resp = requests.get(f"{self.url}/api/v1/health", timeout=0.5)if resp.status_code == 200:- logger.info(f"[AGFSManager] AGFS is ready")+ logger.info("[AGFSManager] AGFS is ready")returnexcept requests.RequestException as e:logger.debug(f"[AGFSManager] Health check failed: {e}")@@ -219,7 +221,9 @@ class AGFSManager:time.sleep(0.1)# Timeout, try reading output- logger.error(f"[AGFSManager] Timeout after {timeout}s, process still running: {self.process.poll() is None}")+ logger.error(+ f"[AGFSManager] Timeout after {timeout}s, process still running: {self.process.poll() is None}"+ )raise TimeoutError(f"AGFS failed to start within {timeout}s")def stop(self) -> None:@@ -241,4 +245,3 @@ class AGFSManager:def is_running(self) -> bool:"""Check if AGFS is running."""return self.process is not None and self.process.poll() is None-diff --git a/openviking/async_client.py b/openviking/async_client.pyindex 8a8975f0a..e1987bc17 100644--- a/openviking/async_client.py+++ b/openviking/async_client.py@@ -10,29 +10,23 @@ from typing import Any, Dict, List, Optional# Local imports - Core modulesfrom openviking.agfs_manager import AGFSManager-from openviking.utils.config import (- OpenVikingConfig,- get_openviking_config,-)-from openviking.utils.uri import VikingURI-from openviking.utils.config.open_viking_config import initialize_openviking_configfrom openviking.core.directories import DirectoryInitializerfrom openviking.session import Sessionfrom openviking.session.compressor import SessionCompressorfrom openviking.storage import VikingDBManagerfrom openviking.storage.collection_schemas import init_context_collection-from openviking.storage.queuefs import get_queue_manager+from openviking.storage.local_fs import export_ovpack as local_export_ovpack+from openviking.storage.local_fs import import_ovpack as local_import_ovpackfrom openviking.storage.observers import QueueObserver, VikingDBObserver, VLMObserver-from openviking.storage.local_fs import (- export_ovpack as local_export_ovpack,- import_ovpack as local_import_ovpack,-)+from openviking.storage.queuefs import get_queue_managerfrom openviking.storage.viking_fs import VikingFS, init_viking_fsfrom openviking.utils import get_logger+from openviking.utils.config import OpenVikingConfig, get_openviking_config+from openviking.utils.config.open_viking_config import initialize_openviking_configfrom openviking.utils.config.storage_config import StorageConfigfrom openviking.utils.resource_processor import ResourceProcessorfrom openviking.utils.skill_processor import SkillProcessor-+from openviking.utils.uri import VikingURIlogger = get_logger(__name__)@@ -597,7 +591,7 @@ class AsyncOpenViking:if not self._viking_fs:raise RuntimeError("VikingFS not initialized")await self._viking_fs.mkdir(uri)-+async def stat(self, uri: str) -> Dict:"""Get resource status"""await self._ensure_initialized()diff --git a/openviking/core/__init__.py b/openviking/core/__init__.pyindex e8c575450..19e15538c 100644--- a/openviking/core/__init__.py+++ b/openviking/core/__init__.py@@ -2,13 +2,13 @@# SPDX-License-Identifier: Apache-2.0"""Core context abstractions for OpenViking."""+from openviking.core.building_tree import BuildingTreefrom openviking.core.context import Context, ResourceContentTypefrom openviking.core.directories import (PRESET_DIRECTORIES,DirectoryDefinition,DirectoryInitializer,)-from openviking.core.building_tree import BuildingTreefrom openviking.core.skill_loader import SkillLoader__all__ = [diff --git a/openviking/core/context.py b/openviking/core/context.pyindex 12c51ee60..6d37fc187 100644--- a/openviking/core/context.py+++ b/openviking/core/context.py@@ -10,11 +10,13 @@ from uuid import uuid4class ResourceContentType(str, Enum):"""Resource content type"""+TEXT = "text"IMAGE = "image"VIDEO = "video"AUDIO = "audio"+class ContextType(str, Enum):"""Context type"""@@ -22,6 +24,7 @@ class ContextType(str, Enum):MEMORY = "memory"RESOURCE = "resource"+class Vectorize:text: str = ""# image: str = ""@@ -31,6 +34,7 @@ class Vectorize:def __init__(self, text: str = ""):self.text = text+class Context:"""Unified context class for all context types in OpenViking.@@ -80,7 +84,7 @@ class Context:return "memory"else:return "resource"-+def _derive_category(self) -> str:"""Derive category from URI prefix."""if self.uri.startswith("viking://agent/memories"):@@ -105,7 +109,7 @@ class Context:def set_vectorize(self, vectorize: Vectorize):self.vectorize = vectorize-+def get_vectorization_text(self) -> str:"""Get text for vectorization."""# todo: multi-modal supportdiff --git a/openviking/core/directories.py b/openviking/core/directories.pyindex d82e564c9..a7e87e575 100644--- a/openviking/core/directories.py+++ b/openviking/core/directories.py@@ -8,10 +8,11 @@ This module defines the preset directory structure that is created on initializa"""from dataclasses import dataclass, field-from datetime import datetime-from typing import TYPE_CHECKING, Any, Dict, List, Optional+from typing import TYPE_CHECKING, Dict, List, Optional+from openviking.core.context import Context, ContextType, Vectorizefrom openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter+if TYPE_CHECKING:from openviking.storage import VikingDBManager@@ -198,8 +199,6 @@ class DirectoryInitializer:count = 1 if created else 0count += await self._initialize_children("user", user_tree.children, user_root_uri)- from openviking.utils.logger import get_logger-return countasync def _ensure_directory(@@ -225,11 +224,11 @@ class DirectoryInitializer:)if not existing:context = Context(- uri=uri,- parent_uri=parent_uri,- is_leaf=False,- context_type=get_context_type_for_uri(uri),- abstract=defn.abstract,+ uri=uri,+ parent_uri=parent_uri,+ is_leaf=False,+ context_type=get_context_type_for_uri(uri),+ abstract=defn.abstract,)context.set_vectorize(Vectorize(text=defn.overview))dir_emb_msg = EmbeddingMsgConverter.from_context(context)@@ -282,6 +281,7 @@ class DirectoryInitializer:async def _create_agfs_structure(self, uri: str, abstract: str, overview: str) -> None:"""Create L0/L1 file structure for directory in AGFS."""from openviking.storage.viking_fs import get_viking_fs+await get_viking_fs().write_context(uri=uri,abstract=abstract,diff --git a/openviking/core/mcp_converter.py b/openviking/core/mcp_converter.pyindex 51dae6113..2faabaa9b 100644--- a/openviking/core/mcp_converter.py+++ b/openviking/core/mcp_converter.py@@ -2,7 +2,7 @@# SPDX-License-Identifier: Apache-2.0"""MCP to Skill converter."""-from typing import Dict, Any+from typing import Any, Dictdef mcp_to_skill(mcp_config: Dict[str, Any]) -> Dict[str, Any]:@@ -42,7 +42,9 @@ def mcp_to_skill(mcp_config: Dict[str, Any]) -> Dict[str, Any]:# Add usage sectionbody_parts.append("\n## Usage\n\n")body_parts.append(f"This tool wraps the MCP tool `{name}`. ")- body_parts.append("Call this when the user needs functionality matching the description above.\n")+ body_parts.append(+ "Call this when the user needs functionality matching the description above.\n"+ )content = "".join(frontmatter_parts) + "".join(body_parts)diff --git a/openviking/message/part.py b/openviking/message/part.pyindex df495e2c0..870debada 100644--- a/openviking/message/part.py+++ b/openviking/message/part.py@@ -6,7 +6,7 @@ Message consists of multiple Parts, each Part has different type and purpose."""from dataclasses import dataclass-from typing import Literal, Union, Optional+from typing import Literal, Optional, Union@dataclassdiff --git a/openviking/models/embedder/__init__.py b/openviking/models/embedder/__init__.pyindex dba382f11..30f459590 100644--- a/openviking/models/embedder/__init__.py+++ b/openviking/models/embedder/__init__.py@@ -14,32 +14,25 @@ Supported providers:"""from openviking.models.embedder.base import (- EmbedResult,- EmbedderBase,+ CompositeHybridEmbedder,DenseEmbedderBase,- SparseEmbedderBase,+ EmbedderBase,+ EmbedResult,HybridEmbedderBase,- CompositeHybridEmbedder,+ SparseEmbedderBase,)--from openviking.models.embedder.openai_embedders import (- OpenAIDenseEmbedder,+from openviking.models.embedder.openai_embedders import OpenAIDenseEmbedder+from openviking.models.embedder.vikingdb_embedders import (+ VikingDBDenseEmbedder,+ VikingDBHybridEmbedder,+ VikingDBSparseEmbedder,)-from openviking.models.embedder.volcengine_embedders import (VolcengineDenseEmbedder,- VolcengineSparseEmbedder,VolcengineHybridEmbedder,+ VolcengineSparseEmbedder,)-from openviking.models.embedder.vikingdb_embedders import (- VikingDBDenseEmbedder,- VikingDBSparseEmbedder,- VikingDBHybridEmbedder,-)---__all__ = [# Base classes"EmbedResult",@@ -48,15 +41,12 @@ __all__ = ["SparseEmbedderBase","HybridEmbedderBase","CompositeHybridEmbedder",-# OpenAI implementations"OpenAIDenseEmbedder",-# Volcengine implementations"VolcengineDenseEmbedder","VolcengineSparseEmbedder","VolcengineHybridEmbedder",-# VikingDB implementations"VikingDBDenseEmbedder","VikingDBSparseEmbedder",diff --git a/openviking/models/embedder/base.py b/openviking/models/embedder/base.pyindex a8cba3bb9..7e7a43690 100644--- a/openviking/models/embedder/base.py+++ b/openviking/models/embedder/base.py@@ -1,8 +1,8 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0from abc import ABC, abstractmethod-from typing import List, Dict, Any, Optionalfrom dataclasses import dataclass+from typing import Any, Dict, List, Optionaldef truncate_and_normalize(embedding: List[float], dimension: Optional[int]) -> List[float]:@@ -19,6 +19,7 @@ def truncate_and_normalize(embedding: List[float], dimension: Optional[int]) ->return embeddingimport math+embedding = embedding[:dimension]norm = math.sqrt(sum(x**2 for x in embedding))if norm > 0:@@ -34,6 +35,7 @@ class EmbedResult:dense_vector: Dense vector in List[float] formatsparse_vector: Sparse vector in Dict[str, float] format, e.g. {'token1': 0.5, 'token2': 0.3}"""+dense_vector: Optional[List[float]] = Nonesparse_vector: Optional[Dict[str, float]] = None@@ -170,7 +172,6 @@ class SparseEmbedderBase(EmbedderBase):return True-class HybridEmbedderBase(EmbedderBase):"""Hybrid embedder base class that returns both dense and sparse vectors@@ -225,9 +226,7 @@ class CompositeHybridEmbedder(HybridEmbedderBase):def __init__(self, dense_embedder: DenseEmbedderBase, sparse_embedder: SparseEmbedderBase):"""Initialize with two separate embedders"""- super().__init__(- model_name=f"{dense_embedder.model_name}+{sparse_embedder.model_name}"- )+ super().__init__(model_name=f"{dense_embedder.model_name}+{sparse_embedder.model_name}")self.dense_embedder = dense_embedderself.sparse_embedder = sparse_embedder@@ -235,22 +234,18 @@ class CompositeHybridEmbedder(HybridEmbedderBase):"""Combine results from both embedders"""dense_res = self.dense_embedder.embed(text)sparse_res = self.sparse_embedder.embed(text)-+return EmbedResult(- dense_vector=dense_res.dense_vector,- sparse_vector=sparse_res.sparse_vector+ dense_vector=dense_res.dense_vector, sparse_vector=sparse_res.sparse_vector)def embed_batch(self, texts: List[str]) -> List[EmbedResult]:"""Combine batch results"""dense_results = self.dense_embedder.embed_batch(texts)sparse_results = self.sparse_embedder.embed_batch(texts)-+return [- EmbedResult(- dense_vector=d.dense_vector,- sparse_vector=s.sparse_vector- )+ EmbedResult(dense_vector=d.dense_vector, sparse_vector=s.sparse_vector)for d, s in zip(dense_results, sparse_results)]@@ -259,4 +254,4 @@ class CompositeHybridEmbedder(HybridEmbedderBase):def close(self):self.dense_embedder.close()- self.sparse_embedder.close()\ No newline at end of file+ self.sparse_embedder.close()diff --git a/openviking/models/embedder/openai_embedders.py b/openviking/models/embedder/openai_embedders.pyindex 75d14fd1c..34f3b5933 100644--- a/openviking/models/embedder/openai_embedders.py+++ b/openviking/models/embedder/openai_embedders.py@@ -1,14 +1,17 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0"""OpenAI Embedder Implementation"""+import os-from typing import List, Optional, Dict, Any+from typing import Any, Dict, List, Optional+import openai+from openviking.models.embedder.base import (DenseEmbedderBase,- SparseEmbedderBase,+ EmbedResult,HybridEmbedderBase,- EmbedResult+ SparseEmbedderBase,)@@ -34,7 +37,7 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):api_key: Optional[str] = None,api_base: Optional[str] = None,dimension: Optional[int] = None,- config: Optional[Dict[str, Any]] = None+ config: Optional[Dict[str, Any]] = None,):"""Initialize OpenAI Dense Embedder@@ -50,7 +53,9 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):"""super().__init__(model_name, config)- self.api_key = api_key or os.getenv("OPENVIKING_EMBEDDING_API_KEY") or os.getenv("OPENAI_API_KEY")+ self.api_key = (+ api_key or os.getenv("OPENVIKING_EMBEDDING_API_KEY") or os.getenv("OPENAI_API_KEY")+ )self.api_base = api_base or os.getenv("OPENVIKING_EMBEDDING_API_BASE")self.dimension = dimension@@ -128,10 +133,7 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):response = self.client.embeddings.create(**kwargs)- return [- EmbedResult(dense_vector=item.embedding)- for item in response.data- ]+ return [EmbedResult(dense_vector=item.embedding) for item in response.data]except openai.APIError as e:raise RuntimeError(f"OpenAI API error: {e.message}") from eexcept Exception as e:diff --git a/openviking/models/embedder/vikingdb_embedders.py b/openviking/models/embedder/vikingdb_embedders.pyindex 3f99a69e4..8ba507b96 100644--- a/openviking/models/embedder/vikingdb_embedders.py+++ b/openviking/models/embedder/vikingdb_embedders.py@@ -1,20 +1,23 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0"""VikingDB Embedder Implementation via HTTP API"""+import os-from typing import List, Optional, Dict, Any+from typing import Any, Dict, List, Optional+from openviking.models.embedder.base import (DenseEmbedderBase,- SparseEmbedderBase,+ EmbedResult,HybridEmbedderBase,- EmbedResult+ SparseEmbedderBase,)from openviking.storage.vectordb.collection.volcengine_clients import ClientForDataApifrom openviking.utils.logger import default_logger as logger+class VikingDBClientMixin:"""Mixin to handle VikingDB Client initialization and API calls."""-+def _init_vikingdb_client(self,ak: Optional[str] = None,@@ -32,15 +35,18 @@ class VikingDBClientMixin:self.client = ClientForDataApi(self.ak, self.sk, self.region, self.host)- def _call_api(self, texts: List[str], dense_model: Dict[str, Any] = None, sparse_model: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]:+ def _call_api(+ self,+ texts: List[str],+ dense_model: Dict[str, Any] = None,+ sparse_model: Optional[Dict[str, Any]] = None,+ ) -> List[Dict[str, Any]]:"""Call VikingDB Embedding API"""path = "/api/vikingdb/embedding"-+data_items = [{"text": text} for text in texts]-- req_body = {- "data": data_items- }++ req_body = {"data": data_items}if dense_model:req_body["dense_model"] = dense_modelif sparse_model:@@ -49,22 +55,27 @@ class VikingDBClientMixin:try:response = self.client.do_req("POST", path, req_body=req_body)if response.status_code != 200:- logger.warning(f"VikingDB API returned bad code: {response.status_code}, message: {response.text}")+ logger.warning(+ f"VikingDB API returned bad code: {response.status_code}, message: {response.text}"+ )return []-+result = response.json()return result.get("result", {}).get("data", [])-+except Exception as e:logger.error(f"Failed to get embeddings: {e}")raise e- def _truncate_and_normalize(self, embedding: List[float], dimension: Optional[int]) -> List[float]:+ def _truncate_and_normalize(+ self, embedding: List[float], dimension: Optional[int]+ ) -> List[float]:"""Truncate and L2 normalize embedding"""if not dimension or len(embedding) <= dimension:return embeddingimport math+embedding = embedding[:dimension]norm = math.sqrt(sum(x**2 for x in embedding))if norm > 0:@@ -79,7 +90,7 @@ class VikingDBClientMixin:result = {}if isinstance(sparse_data, dict):return {str(k): float(v) for k, v in sparse_data.items()}-+if isinstance(sparse_data, list):for item in sparse_data:if isinstance(item, dict):@@ -93,7 +104,7 @@ class VikingDBClientMixin:class VikingDBDenseEmbedder(DenseEmbedderBase, VikingDBClientMixin):"""VikingDB Dense Embedder"""-+def __init__(self,model_name: str,@@ -104,29 +115,25 @@ class VikingDBDenseEmbedder(DenseEmbedderBase, VikingDBClientMixin):host: Optional[str] = None,dimension: Optional[int] = None,embedding_type: str = "text",- config: Optional[Dict[str, Any]] = None+ config: Optional[Dict[str, Any]] = None,):DenseEmbedderBase.__init__(self, model_name, config)self._init_vikingdb_client(ak, sk, region, host)self.model_version = model_versionself.dimension = dimensionself.embedding_type = embedding_type- self.dense_model = {- "name": model_name,- "version": model_version,- "dim": dimension- }+ self.dense_model = {"name": model_name, "version": model_version, "dim": dimension}def embed(self, text: str) -> EmbedResult:results = self._call_api([text], dense_model=self.dense_model)if not results:return EmbedResult(dense_vector=[])-+item = results[0]dense_vector = []if "dense_embedding" in item:dense_vector = self._truncate_and_normalize(item["dense_embedding"], self.dimension)-+return EmbedResult(dense_vector=dense_vector)def embed_batch(self, texts: List[str]) -> List[EmbedResult]:@@ -134,17 +141,21 @@ class VikingDBDenseEmbedder(DenseEmbedderBase, VikingDBClientMixin):return []raw_results = self._call_api(texts, dense_model=self.dense_model)return [- EmbedResult(dense_vector=self._truncate_and_normalize(item.get("dense_embedding", []), self.dimension))+ EmbedResult(+ dense_vector=self._truncate_and_normalize(+ item.get("dense_embedding", []), self.dimension+ )+ )for item in raw_results]-+def get_dimension(self) -> int:return self.dimension if self.dimension else 2048class VikingDBSparseEmbedder(SparseEmbedderBase, VikingDBClientMixin):"""VikingDB Sparse Embedder"""-+def __init__(self,model_name: str,@@ -153,7 +164,7 @@ class VikingDBSparseEmbedder(SparseEmbedderBase, VikingDBClientMixin):sk: Optional[str] = None,region: Optional[str] = None,host: Optional[str] = None,- config: Optional[Dict[str, Any]] = None+ config: Optional[Dict[str, Any]] = None,):SparseEmbedderBase.__init__(self, model_name, config)self._init_vikingdb_client(ak, sk, region, host)@@ -167,12 +178,12 @@ class VikingDBSparseEmbedder(SparseEmbedderBase, VikingDBClientMixin):results = self._call_api([text], sparse_model=self.sparse_model)if not results:return EmbedResult(sparse_vector={})-+item = results[0]sparse_vector = {}if "sparse" in item:sparse_vector = item["sparse"]-+return EmbedResult(sparse_vector=sparse_vector)def embed_batch(self, texts: List[str]) -> List[EmbedResult]:@@ -180,14 +191,16 @@ class VikingDBSparseEmbedder(SparseEmbedderBase, VikingDBClientMixin):return []raw_results = self._call_api(texts, sparse_model=self.sparse_model)return [- EmbedResult(sparse_vector=self._process_sparse_embedding(item.get("sparse_embedding", {})))+ EmbedResult(+ sparse_vector=self._process_sparse_embedding(item.get("sparse_embedding", {}))+ )for item in raw_results]class VikingDBHybridEmbedder(HybridEmbedderBase, VikingDBClientMixin):"""VikingDB Hybrid Embedder"""-+def __init__(self,model_name: str,@@ -198,29 +211,26 @@ class VikingDBHybridEmbedder(HybridEmbedderBase, VikingDBClientMixin):host: Optional[str] = None,dimension: Optional[int] = None,embedding_type: str = "text",- config: Optional[Dict[str, Any]] = None+ config: Optional[Dict[str, Any]] = None,):HybridEmbedderBase.__init__(self, model_name, config)self._init_vikingdb_client(ak, sk, region, host)self.model_version = model_versionself.dimension = dimensionself.embedding_type = embedding_type- self.dense_model = {- "name": model_name,- "version": model_version,- "dim": dimension- }+ self.dense_model = {"name": model_name, "version": model_version, "dim": dimension}self.sparse_model = {"name": model_name,"version": model_version,}-def embed(self, text: str) -> EmbedResult:- results = self._call_api([text], dense_model=self.dense_model, sparse_model=self.sparse_model)+ results = self._call_api(+ [text], dense_model=self.dense_model, sparse_model=self.sparse_model+ )if not results:return EmbedResult(dense_vector=[], sparse_vector={})-+item = results[0]dense_vector = []sparse_vector = {}@@ -229,13 +239,15 @@ class VikingDBHybridEmbedder(HybridEmbedderBase, VikingDBClientMixin):dense_vector = self._truncate_and_normalize(item["dense"], self.dimension)if "sparse" in item:sparse_vector = item["sparse"]-+return EmbedResult(dense_vector=dense_vector, sparse_vector=sparse_vector)def embed_batch(self, texts: List[str]) -> List[EmbedResult]:if not texts:return []- raw_results = self._call_api(texts, dense_model=self.dense_model, sparse_model=self.sparse_model)+ raw_results = self._call_api(+ texts, dense_model=self.dense_model, sparse_model=self.sparse_model+ )results = []for item in raw_results:if "dense" in item:diff --git a/openviking/models/embedder/volcengine_embedders.py b/openviking/models/embedder/volcengine_embedders.pyindex 6cd764213..5c3ca0d28 100644--- a/openviking/models/embedder/volcengine_embedders.py+++ b/openviking/models/embedder/volcengine_embedders.py@@ -1,15 +1,18 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0"""Volcengine Embedder Implementation"""+import os-from typing import List, Optional, Dict, Any, Union+from typing import Any, Dict, List, Optional+import volcenginesdkarkruntime+from openviking.models.embedder.base import (DenseEmbedderBase,- SparseEmbedderBase,- HybridEmbedderBase,EmbedResult,- truncate_and_normalize+ HybridEmbedderBase,+ SparseEmbedderBase,+ truncate_and_normalize,)@@ -18,17 +21,17 @@ def process_sparse_embedding(sparse_data: Any) -> Dict[str, float]:if not sparse_data:return {}result = {}-+# Helper to extract index/value from an item (dict or object)def extract_pair(item):- idx = getattr(item, 'index', None)+ idx = getattr(item, "index", None)if idx is None and isinstance(item, dict):- idx = item.get('index')-- val = getattr(item, 'value', None)+ idx = item.get("index")++ val = getattr(item, "value", None)if val is None and isinstance(item, dict):- val = item.get('value')-+ val = item.get("value")+return idx, valif isinstance(sparse_data, list):@@ -36,14 +39,14 @@ def process_sparse_embedding(sparse_data: Any) -> Dict[str, float]:idx, val = extract_pair(item)if idx is not None and val is not None:result[str(idx)] = float(val)- elif hasattr(sparse_data, 'index'):- # Single object case (unlikely for vector but possible per type hint)- idx, val = extract_pair(sparse_data)- if idx is not None and val is not None:- result[str(idx)] = float(val)+ elif hasattr(sparse_data, "index"):+ # Single object case (unlikely for vector but possible per type hint)+ idx, val = extract_pair(sparse_data)+ if idx is not None and val is not None:+ result[str(idx)] = float(val)elif isinstance(sparse_data, dict):# Maybe a direct dict?- return {str(k): float(v) for k, v in sparse_data.items()}+ return {str(k): float(v) for k, v in sparse_data.items()}return result@@ -61,7 +64,7 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):api_base: Optional[str] = None,dimension: Optional[int] = None,input_type: str = "multimodal",- config: Optional[Dict[str, Any]] = None+ config: Optional[Dict[str, Any]] = None,):"""Initialize Volcengine Dense Embedder@@ -78,8 +81,14 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):"""super().__init__(model_name, config)- self.api_key = api_key or os.getenv("OPENVIKING_EMBEDDING_API_KEY") or os.getenv("VOLC_API_KEY")- self.api_base = api_base or os.getenv("OPENVIKING_EMBEDDING_API_BASE") or "https://ark.cn-beijing.volces.com/api/v3"+ self.api_key = (+ api_key or os.getenv("OPENVIKING_EMBEDDING_API_KEY") or os.getenv("VOLC_API_KEY")+ )+ self.api_base = (+ api_base+ or os.getenv("OPENVIKING_EMBEDDING_API_BASE")+ or "https://ark.cn-beijing.volces.com/api/v3"+ )self.dimension = dimensionself.input_type = input_type@@ -87,10 +96,7 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):raise ValueError("api_key is required")# Initialize Volcengine client- self.client = volcenginesdkarkruntime.Ark(- api_key=self.api_key,- base_url=self.api_base- )+ self.client = volcenginesdkarkruntime.Ark(api_key=self.api_key, base_url=self.api_base)# Auto-detect dimensionself._dimension = dimension@@ -121,16 +127,12 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):if self.input_type == "multimodal":# Use multimodal embeddings APIresponse = self.client.multimodal_embeddings.create(- input=[{'type': 'text', 'text': text}],- model=self.model_name+ input=[{"type": "text", "text": text}], model=self.model_name)vector = response.data.embeddingelse:# Use text embeddings API- response = self.client.embeddings.create(- input=text,- model=self.model_name- )+ response = self.client.embeddings.create(input=text, model=self.model_name)vector = response.data[0].embeddingvector = truncate_and_normalize(vector, self.dimension)@@ -155,17 +157,13 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):try:if self.input_type == "multimodal":- multimodal_inputs = [{'type': 'text', 'text': text} for text in texts]+ multimodal_inputs = [{"type": "text", "text": text} for text in texts]response = self.client.multimodal_embeddings.create(- input=multimodal_inputs,- model=self.model_name+ input=multimodal_inputs, model=self.model_name)data = response.dataelse:- response = self.client.embeddings.create(- input=texts,- model=self.model_name- )+ response = self.client.embeddings.create(input=texts, model=self.model_name)data = response.datareturn [@@ -190,7 +188,7 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):model_name: str,api_key: Optional[str] = None,api_base: Optional[str] = None,- config: Optional[Dict[str, Any]] = None+ config: Optional[Dict[str, Any]] = None,):"""Initialize Volcengine Sparse Embedder@@ -206,17 +204,12 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):super().__init__(model_name, config)self.api_key = api_key- self.api_base = api_base-+ self.api_base = api_baseif not self.api_key:raise ValueError("api_key is required")- self.client = volcenginesdkarkruntime.Ark(- api_key=self.api_key,- base_url=self.api_base- )-+ self.client = volcenginesdkarkruntime.Ark(api_key=self.api_key, base_url=self.api_base)def embed(self, text: str) -> EmbedResult:"""Perform sparse embedding on text@@ -233,12 +226,12 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):try:# Must use multimodal endpoint for sparseresponse = self.client.multimodal_embeddings.create(- input=[{'type': 'text', 'text': text}],+ input=[{"type": "text", "text": text}],model=self.model_name,- sparse_embedding={'type': 'enabled'}+ sparse_embedding={"type": "enabled"},)item = response.data[0]- sparse_vector = getattr(item, 'sparse_embedding', None)+ sparse_vector = getattr(item, "sparse_embedding", None)return EmbedResult(sparse_vector=process_sparse_embedding(sparse_vector))except Exception as e:raise RuntimeError(f"Volcengine sparse embedding failed: {str(e)}") from e@@ -255,17 +248,16 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):Raises:RuntimeError: When API call fails"""- if not texts: return []+ if not texts:+ return []try:- multimodal_inputs = [{'type': 'text', 'text': text} for text in texts]+ multimodal_inputs = [{"type": "text", "text": text} for text in texts]response = self.client.multimodal_embeddings.create(- input=multimodal_inputs,- model=self.model_name,- sparse_embedding={'type': 'enabled'}+ input=multimodal_inputs, model=self.model_name, sparse_embedding={"type": "enabled"})results = []for item in response.data:- sparse_vector = getattr(item, 'sparse_embedding', None)+ sparse_vector = getattr(item, "sparse_embedding", None)results.append(EmbedResult(sparse_vector=process_sparse_embedding(sparse_vector)))return resultsexcept Exception as e:@@ -286,7 +278,7 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):api_base: Optional[str] = None,dimension: Optional[int] = None,input_type: str = "multimodal",- config: Optional[Dict[str, Any]] = None+ config: Optional[Dict[str, Any]] = None,):"""Initialize Volcengine Hybrid Embedder@@ -310,10 +302,7 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):if not self.api_key:raise ValueError("api_key is required")- self.client = volcenginesdkarkruntime.Ark(- api_key=self.api_key,- base_url=self.api_base- )+ self.client = volcenginesdkarkruntime.Ark(api_key=self.api_key, base_url=self.api_base)self._dimension = dimension or 2048def embed(self, text: str) -> EmbedResult:@@ -332,17 +321,16 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):# Always use multimodal for hybrid to get bothresponse = self.client.multimodal_embeddings.create(- input=[{'type': 'text', 'text': text}],+ input=[{"type": "text", "text": text}],model=self.model_name,- sparse_embedding={'type': 'enabled'}+ sparse_embedding={"type": "enabled"},)item = response.datadense_vector = truncate_and_normalize(item.embedding, self.dimension)- sparse_vector = getattr(item, 'sparse_embedding', None)+ sparse_vector = getattr(item, "sparse_embedding", None)return EmbedResult(- dense_vector=dense_vector,- sparse_vector=process_sparse_embedding(sparse_vector)+ dense_vector=dense_vector, sparse_vector=process_sparse_embedding(sparse_vector))except Exception as e:raise RuntimeError(f"Volcengine hybrid embedding failed: {str(e)}") from e@@ -359,22 +347,23 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):Raises:RuntimeError: When API call fails"""- if not texts: return []+ if not texts:+ return []try:- multimodal_inputs = [{'type': 'text', 'text': text} for text in texts]+ multimodal_inputs = [{"type": "text", "text": text} for text in texts]response = self.client.multimodal_embeddings.create(- input=multimodal_inputs,- model=self.model_name,- sparse_embedding={'type': 'enabled'}+ input=multimodal_inputs, model=self.model_name, sparse_embedding={"type": "enabled"})results = []for item in response.data:dense_vector = truncate_and_normalize(item.embedding, self.dimension)- sparse_vector = getattr(item, 'sparse_embedding', None)- results.append(EmbedResult(- dense_vector=dense_vector,- sparse_vector=process_sparse_embedding(sparse_vector)- ))+ sparse_vector = getattr(item, "sparse_embedding", None)+ results.append(+ EmbedResult(+ dense_vector=dense_vector,+ sparse_vector=process_sparse_embedding(sparse_vector),+ )+ )return resultsexcept Exception as e:raise RuntimeError(f"Volcengine batch hybrid embedding failed: {str(e)}") from ediff --git a/openviking/models/vlm/__init__.py b/openviking/models/vlm/__init__.pyindex 598d126ec..8e38aceae 100644--- a/openviking/models/vlm/__init__.py+++ b/openviking/models/vlm/__init__.py@@ -2,9 +2,9 @@# SPDX-License-Identifier: Apache-2.0"""VLM (Vision-Language Model) module"""-from .base import VLMBase, VLMFactoryfrom .backends.openai_vlm import OpenAIVLMfrom .backends.volcengine_vlm import VolcEngineVLM+from .base import VLMBase, VLMFactory__all__ = ["VLMBase",diff --git a/openviking/models/vlm/backends/openai_vlm.py b/openviking/models/vlm/backends/openai_vlm.pyindex 14f8b1939..128da40bb 100644--- a/openviking/models/vlm/backends/openai_vlm.py+++ b/openviking/models/vlm/backends/openai_vlm.py@@ -2,10 +2,10 @@# SPDX-License-Identifier: Apache-2.0"""OpenAI VLM backend implementation"""-import base64import asyncio+import base64from pathlib import Path-from typing import Any, Dict, List, Optional, Union+from typing import Any, Dict, List, Unionfrom ..base import VLMBase@@ -63,12 +63,14 @@ class OpenAIVLM(VLMBase):if self.provider == "volcengine":kwargs["thinking"] = {"type": "disabled" if not thinking else "enabled"}-+response = client.chat.completions.create(**kwargs)self._update_token_usage_from_response(response)return response.choices[0].message.content or ""- async def get_completion_async(self, prompt: str, thinking: bool = False, max_retries: int = 0) -> str:+ async def get_completion_async(+ self, prompt: str, thinking: bool = False, max_retries: int = 0+ ) -> str:"""Get text completion asynchronously"""client = self.get_async_client()kwargs = {diff --git a/openviking/models/vlm/backends/volcengine_vlm.py b/openviking/models/vlm/backends/volcengine_vlm.pyindex 8f1d2e97c..f11a289e6 100644--- a/openviking/models/vlm/backends/volcengine_vlm.py+++ b/openviking/models/vlm/backends/volcengine_vlm.py@@ -2,13 +2,12 @@# SPDX-License-Identifier: Apache-2.0"""VolcEngine VLM backend implementation"""-import base64from pathlib import Path-from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING+from typing import Any, Dict, List, Union-from ..base import VLMBasefrom .openai_vlm import OpenAIVLM+class VolcEngineVLM(OpenAIVLM):"""VolcEngine VLM backend"""@@ -74,4 +73,3 @@ class VolcEngineVLM(OpenAIVLM):images: List[Union[str, Path, bytes]],) -> str:return await super().get_vision_completion_async(prompt, images)-diff --git a/openviking/models/vlm/base.py b/openviking/models/vlm/base.pyindex 80dfacc13..25d63aa01 100644--- a/openviking/models/vlm/base.py+++ b/openviking/models/vlm/base.py@@ -4,7 +4,7 @@from abc import ABC, abstractmethodfrom pathlib import Path-from typing import Any, Dict, List, Optional, Union+from typing import Any, Dict, List, Unionfrom .token_usage import TokenUsageTracker@@ -120,12 +120,14 @@ class VLMFactory:ImportError: If related dependencies are not installed"""provider = config.get("provider") or config.get("backend") or "openai"-+if provider == "openai":from .backends.openai_vlm import OpenAIVLM+return OpenAIVLM(config)elif provider == "volcengine":from .backends.volcengine_vlm import VolcEngineVLM+return VolcEngineVLM(config)else:raise ValueError(f"Unsupported VLM provider: {provider}")diff --git a/openviking/models/vlm/llm.py b/openviking/models/vlm/llm.pyindex c59c2a8b6..f657f0304 100644--- a/openviking/models/vlm/llm.py+++ b/openviking/models/vlm/llm.py@@ -10,10 +10,10 @@ import jsonimport refrom typing import Any, Dict, Optional, Type, TypeVar+import json_repairfrom pydantic import BaseModelfrom openviking.utils.logger import get_logger-import json_repairlogger = get_logger(__name__)diff --git a/openviking/models/vlm/token_usage.py b/openviking/models/vlm/token_usage.pyindex f1b511946..3df8db6ab 100644--- a/openviking/models/vlm/token_usage.py+++ b/openviking/models/vlm/token_usage.py@@ -111,7 +111,10 @@ class ModelTokenUsage:def __str__(self) -> str:providers = ", ".join(- [f"{provider}: {usage.total_tokens}" for provider, usage in self.usage_by_provider.items()]+ [+ f"{provider}: {usage.total_tokens}"+ for provider, usage in self.usage_by_provider.items()+ ])return f"ModelTokenUsage(model={self.model_name}, total={self.total_usage.total_tokens}, providers=[{providers}])"diff --git a/openviking/parse/__init__.py b/openviking/parse/__init__.pyindex 0a8414d21..c6dff700c 100644--- a/openviking/parse/__init__.py+++ b/openviking/parse/__init__.py@@ -2,18 +2,9 @@# SPDX-License-Identifier: Apache-2.0"""Document parsers for various formats."""-from openviking.parse.base import (- NodeType,- ParseResult,- ResourceNode,- create_parse_result,-)+from openviking.parse.base import NodeType, ParseResult, ResourceNode, create_parse_resultfrom openviking.parse.converter import DocumentConverter-from openviking.parse.custom import (- CallbackParserWrapper,- CustomParserProtocol,- CustomParserWrapper,-)+from openviking.parse.custom import CallbackParserWrapper, CustomParserProtocol, CustomParserWrapperfrom openviking.parse.parsers.base_parser import BaseParserfrom openviking.parse.parsers.code import CodeParserfrom openviking.parse.parsers.html import HTMLParserdiff --git a/openviking/parse/base.py b/openviking/parse/base.pyindex 71de87976..9548f4b4c 100644--- a/openviking/parse/base.py+++ b/openviking/parse/base.py@@ -7,15 +7,14 @@ Following PageIndex philosophy: preserve natural document structurerather than arbitrary chunking."""-from abc import ABC, abstractmethodfrom dataclasses import dataclass, fieldfrom datetime import datetimefrom enum import Enumfrom pathlib import Path-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union+from typing import TYPE_CHECKING, Any, Dict, List, Optionalif TYPE_CHECKING:- from openviking.storage.viking_fs import VikingFS+ pass# ============================================================================# Common utility functions@@ -97,7 +96,9 @@ def lazy_import(module_name: str, package_name: Optional[str] = None) -> Any:return importlib.import_module(module_name)except ImportError:pkg = package_name or module_name- raise ImportError(f"Module '{module_name}' not available. Please install: pip install {pkg}")+ raise ImportError(+ f"Module '{module_name}' not available. Please install: pip install {pkg}"+ )class ResourceCategory(Enum):@@ -154,6 +155,7 @@ class NodeType(Enum):ROOT = "root"SECTION = "section"+@dataclassclass ResourceNode:"""@@ -200,6 +202,7 @@ class ResourceNode:async def get_detail_content_async(self, temp_uri: str) -> str:"""Read detail file content from VikingFS temp directory."""from openviking.storage.viking_fs import get_viking_fs+if not self.detail_file:return ""file_uri = f"{temp_uri}/{self.detail_file}"diff --git a/openviking/parse/custom.py b/openviking/parse/custom.pyindex 951569c7e..78636a9fc 100644--- a/openviking/parse/custom.py+++ b/openviking/parse/custom.py@@ -131,8 +131,7 @@ class CustomParserWrapper:"""if not self.custom_parser.can_handle(source):raise ValueError(- f"Parser {type(self.custom_parser).__name__} "- f"cannot handle source: {source}"+ f"Parser {type(self.custom_parser).__name__} cannot handle source: {source}")return await self.custom_parser.parse(source, **kwargs)diff --git a/openviking/parse/parsers/base_parser.py b/openviking/parse/parsers/base_parser.pyindex 89d5766b1..4345d6232 100644--- a/openviking/parse/parsers/base_parser.py+++ b/openviking/parse/parsers/base_parser.py@@ -3,6 +3,7 @@from abc import ABC, abstractmethodfrom pathlib import Pathfrom typing import List, Optional, Union+from openviking.parse.base import ParseResultdiff --git a/openviking/parse/parsers/code.py b/openviking/parse/parsers/code.pyindex 65c16b97f..777ee9173 100644--- a/openviking/parse/parsers/code.py+++ b/openviking/parse/parsers/code.py@@ -7,10 +7,10 @@ Supports multiple programming languages with syntax-aware parsing."""from pathlib import Path-from typing import Any, Dict, List, Optional, Union+from typing import List, Optional, Union+from openviking.parse.base import ParseResultfrom openviking.parse.parsers.base_parser import BaseParser-from openviking.parse.base import NodeType, ParseResult, ResourceNodefrom openviking.utils.config.parser_config import CodeConfigfrom openviking.utils.logger import get_logger@@ -164,4 +164,4 @@ class CodeParser(BaseParser):Raises:ValueError: If content is empty"""- pass\ No newline at end of file+ passdiff --git a/openviking/parse/parsers/html.py b/openviking/parse/parsers/html.pyindex 4549c288b..36f1e68ef 100644--- a/openviking/parse/parsers/html.py+++ b/openviking/parse/parsers/html.py@@ -18,7 +18,7 @@ from enum import Enumfrom pathlib import Pathfrom typing import Any, Dict, List, Optional, Tuple, Unionfrom urllib.parse import urlparse-from openviking.parse.parsers.base_parser import BaseParser+from openviking.parse.base import (NodeType,ParseResult,@@ -26,6 +26,7 @@ from openviking.parse.base import (create_parse_result,lazy_import,)+from openviking.parse.parsers.base_parser import BaseParserclass URLType(Enum):@@ -68,9 +69,7 @@ class URLTypeDetector:"application/xhtml+xml": URLType.WEBPAGE,}- async def detect(- self, url: str, timeout: float = 10.0- ) -> Tuple[URLType, Dict[str, Any]]:+ async def detect(self, url: str, timeout: float = 10.0) -> Tuple[URLType, Dict[str, Any]]:"""Detect URL content type.@@ -157,6 +156,7 @@ class HTMLParser(BaseParser):if not hasattr(self, "_readabilipy") or self._readabilipy is None:try:from readabilipy import simple_json+self._readabilipy = simple_jsonexcept ImportError:raise ImportError(@@ -170,6 +170,7 @@ class HTMLParser(BaseParser):if not hasattr(self, "_markdownify") or self._markdownify is None:try:import markdownify+self._markdownify = markdownifyexcept ImportError:raise ImportError(@@ -400,9 +401,7 @@ class HTMLParser(BaseParser):"""httpx = lazy_import("httpx")- async with httpx.AsyncClient(- timeout=self.timeout, follow_redirects=True- ) as client:+ async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:headers = {"User-Agent": self.user_agent}response = await client.get(url, headers=headers)response.raise_for_status()@@ -433,9 +432,7 @@ class HTMLParser(BaseParser):temp_file.close()# Download- async with httpx.AsyncClient(- timeout=self.timeout, follow_redirects=True- ) as client:+ async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:headers = {"User-Agent": self.user_agent}response = await client.get(url, headers=headers)response.raise_for_status()diff --git a/openviking/parse/parsers/markdown.py b/openviking/parse/parsers/markdown.pyindex 53f7666dd..6620130ea 100644--- a/openviking/parse/parsers/markdown.py+++ b/openviking/parse/parsers/markdown.py@@ -17,21 +17,13 @@ The parser handles scenarios:5. Oversized sections without subsections β split by paragraphs"""-import jsonimport reimport time-import uuidfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union-from openviking.utils.config import get_openviking_config+from openviking.parse.base import NodeType, ParseResult, ResourceNode, create_parse_resultfrom openviking.parse.parsers.base_parser import BaseParser-from openviking.parse.base import (- NodeType,- ParseResult,- ResourceNode,- create_parse_result,-)from openviking.utils.config.parser_config import ParserConfigfrom openviking.utils.logger import get_logger@@ -382,9 +374,7 @@ class MarkdownParser(BaseParser):await viking_fs.mkdir(root_dir)# Get document name- doc_name = self._sanitize_for_path(- Path(source_path).stem if source_path else "content"- )+ doc_name = self._sanitize_for_path(Path(source_path).stem if source_path else "content")# Small document: save as single fileif estimated_tokens <= max_size:@@ -410,22 +400,26 @@ class MarkdownParser(BaseParser):pre_content = content[:first_heading_start].strip()if pre_content:pre_tokens = self._estimate_token_count(pre_content)- sections.append({- "name": doc_name,- "content": pre_content,- "tokens": pre_tokens,- "has_children": False,- "heading_idx": None,- })+ sections.append(+ {+ "name": doc_name,+ "content": pre_content,+ "tokens": pre_tokens,+ "has_children": False,+ "heading_idx": None,+ }+ )# Add real sections (top-level only for this pass)min_level = min(h[3] for h in headings)i = 0while i < len(headings):if headings[i][3] == min_level:- sections.append({- "heading_idx": i,- })+ sections.append(+ {+ "heading_idx": i,+ }+ )i += 1# Process sections with merge logic@@ -448,7 +442,8 @@ class MarkdownParser(BaseParser):# Expand section infoexpanded = [- section if section.get("heading_idx") is None+ section+ if section.get("heading_idx") is Noneelse self._get_section_info(content, headings, section["heading_idx"])for section in sections]@@ -481,10 +476,7 @@ class MarkdownParser(BaseParser):def _can_merge(self, pending: List, tokens: int, max_size: int, has_children: bool) -> bool:"""Check if section can merge with pending."""- return (- sum(t for _, _, t in pending) + tokens <= max_size- and not has_children- )+ return sum(t for _, _, t in pending) + tokens <= max_size and not has_childrenasync def _try_add_to_pending(self, viking_fs, parent_dir: str, pending: List, item: Tuple, max_size: int@@ -547,13 +539,15 @@ class MarkdownParser(BaseParser):"""Build and process child sections."""children = []if section.get("direct_content"):- children.append({- "name": name,- "content": section["direct_content"],- "tokens": self._estimate_token_count(section["direct_content"]),- "has_children": False,- "heading_idx": None,- })+ children.append(+ {+ "name": name,+ "content": section["direct_content"],+ "tokens": self._estimate_token_count(section["direct_content"]),+ "has_children": False,+ "heading_idx": None,+ }+ )for child_idx in section.get("child_indices", []):children.append({"heading_idx": child_idx})@@ -650,4 +644,3 @@ class MarkdownParser(BaseParser):cjk_chars = len(re.findall(r"[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]", content))other_chars = len(re.findall(r"[^\s]", content)) - cjk_charsreturn int(cjk_chars * 0.7 + other_chars * 0.3)-diff --git a/openviking/parse/parsers/media.py b/openviking/parse/parsers/media.pyindex 8a38307ab..9e2e5dbca 100644--- a/openviking/parse/parsers/media.py+++ b/openviking/parse/parsers/media.py@@ -10,17 +10,14 @@ They serve as a design reference for future media parsing capabilities.For current document parsing (PDF, Markdown, HTML, Text), see other parser modules."""-from PIL import Imagefrom pathlib import Pathfrom typing import List, Optional, Union-from openviking.parse.parsers.base_parser import BaseParser-from openviking.parse.base import NodeType, ParseResult, ResourceNode-from openviking.utils.config.parser_config import (- AudioConfig,- ImageConfig,- VideoConfig,-)+from PIL import Image++from openviking.parse.base import NodeType, ParseResult, ResourceNode+from openviking.parse.parsers.base_parser import BaseParser+from openviking.utils.config.parser_config import AudioConfig, ImageConfig, VideoConfig# =============================================================================# Configuration Classes@@ -117,7 +114,7 @@ class ImageParser(BaseParser):img = Image.open(file_path)width, height = img.sizeformat_str = img.format or ext[1:].upper()- except Exception as e:+ except Exception:width, height = 0, 0format_str = ext[1:].upper()@@ -183,7 +180,7 @@ class ImageParser(BaseParser):TODO: Integrate with actual VLM API (OpenAI GPT-4V, Claude Vision, etc.)"""# Fallback implementation - returns basic placeholder- return f"Image description (VLM integration pending)\n\nThis is an image. VLM description feature has not yet integrated external API."+ return "Image description (VLM integration pending)\n\nThis is an image. VLM description feature has not yet integrated external API."async def _ocr_extract(self, image_bytes: bytes, lang: str) -> Optional[str]:"""diff --git a/openviking/parse/parsers/pdf.py b/openviking/parse/parsers/pdf.pyindex 1c6397a05..7ada2c784 100644--- a/openviking/parse/parsers/pdf.py+++ b/openviking/parse/parsers/pdf.py@@ -16,7 +16,7 @@ import loggingimport timefrom pathlib import Pathfrom typing import Any, Dict, List, Optional, Union-from openviking.parse.parsers.base_parser import BaseParser+from openviking.parse.base import (NodeType,ParseResult,@@ -24,6 +24,7 @@ from openviking.parse.base import (create_parse_result,lazy_import,)+from openviking.parse.parsers.base_parser import BaseParserfrom openviking.utils.config.parser_config import PDFConfiglogger = logging.getLogger(__name__)diff --git a/openviking/parse/parsers/text.py b/openviking/parse/parsers/text.pyindex ef559710e..902614315 100644--- a/openviking/parse/parsers/text.py+++ b/openviking/parse/parsers/text.py@@ -8,8 +8,9 @@ Delegates to MarkdownParser since plain text is just unformatted markdown.from pathlib import Pathfrom typing import List, Optional, Union-from openviking.parse.parsers.base_parser import BaseParser+from openviking.parse.base import ParseResult+from openviking.parse.parsers.base_parser import BaseParserfrom openviking.utils.config.parser_config import ParserConfigdiff --git a/openviking/parse/registry.py b/openviking/parse/registry.pyindex c9acafbbe..023fb1ad0 100644--- a/openviking/parse/registry.py+++ b/openviking/parse/registry.py@@ -8,15 +8,15 @@ Provides automatic parser selection based on file type.import loggingfrom pathlib import Path-from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Type, Union+from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Unionfrom openviking.parse.base import ParseResultfrom openviking.parse.parsers.base_parser import BaseParser+from openviking.parse.parsers.code import CodeParserfrom openviking.parse.parsers.html import HTMLParserfrom openviking.parse.parsers.markdown import MarkdownParserfrom openviking.parse.parsers.pdf import PDFParserfrom openviking.parse.parsers.text import TextParser-from openviking.parse.parsers.code import CodeParserif TYPE_CHECKING:from openviking.parse.custom import CustomParserProtocol@@ -53,6 +53,7 @@ class ParserRegistry:if register_optional:try:from openviking.parse.parsers.media import ImageParser+self.register("image", ImageParser())logger.info("Registered ImageParser for image formats")except ImportError as e:diff --git a/openviking/parse/resource_detector/__init__.py b/openviking/parse/resource_detector/__init__.pyindex 48e37587e..2a6416107 100644--- a/openviking/parse/resource_detector/__init__.py+++ b/openviking/parse/resource_detector/__init__.py@@ -1,4 +1,3 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0"""Detect resource type and understand the content before process."""-diff --git a/openviking/parse/resource_detector/detect_info.py b/openviking/parse/resource_detector/detect_info.pyindex 2c301163c..30135cd93 100644--- a/openviking/parse/resource_detector/detect_info.py+++ b/openviking/parse/resource_detector/detect_info.py@@ -7,6 +7,10 @@ We need to know:"""+from dataclasses import dataclass+from enum import Enum++class VisitType(Enum):# Direct content that can be used immediately, e.g., strings containing conversation content, JSON content, etc.DIRECT_CONTENT = "DIRECT_CONTENT"@@ -17,6 +21,7 @@ class VisitType(Enum):# Pre-processed context pack conforming to OpenViking's structure, typically with .ovpack extensionREADY_CONTEXT_PACK = "READY_CONTEXT_PACK"+class SizeType(Enum):# Content can be processed directly in memory, e.g., small text segmentsIN_MEM = "IN_MEM"@@ -25,6 +30,7 @@ class SizeType(Enum):# Content too large to process, e.g., exceeds X GB, may cause system crash or performance issuesTOO_LARGE_TO_PROCESS = "TOO_LARGE_TO_PROCESS"+class RecursiveType(Enum):# Single file, no recursive processing requiredSINGLE = "SINGLE"@@ -33,6 +39,7 @@ class RecursiveType(Enum):# Files that need to be expanded for recursive processing, e.g., compressed files, READY_CONTEXT_PACK, etc.EXPAND_TO_RECURSIVE = "EXPAND_TO_RECURSIVE"+@dataclassclass DetectInfo:visit_type: VisitTypediff --git a/openviking/parse/tree_builder.py b/openviking/parse/tree_builder.pyindex 7d0960328..7bb6d4f9e 100644--- a/openviking/parse/tree_builder.py+++ b/openviking/parse/tree_builder.py@@ -21,12 +21,11 @@ IMPORTANT (v5.0 Architecture):"""import logging-from typing import TYPE_CHECKING, List, Optional, Tuple-from openviking.utils.config import get_openviking_config-from openviking.utils.uri import VikingURI+from typing import TYPE_CHECKING, Optional+from openviking.core.building_tree import BuildingTree-from openviking.prompts import render_promptfrom openviking.storage.viking_fs import get_viking_fs+from openviking.utils.uri import VikingURIif TYPE_CHECKING:pass@@ -206,9 +205,7 @@ class TreeBuilder:if "exist" not in str(e).lower():logger.debug(f"Parent dir {parent_uri} may already exist: {e}")- async def _enqueue_semantic_generation(- self, uri: str, context_type: str- ) -> None:+ async def _enqueue_semantic_generation(self, uri: str, context_type: str) -> None:"""Enqueue a directory for semantic generation.@@ -216,16 +213,12 @@ class TreeBuilder:uri: Directory URI to enqueuecontext_type: resource/memory/skill"""- from openviking.storage.queuefs import get_queue_manager, SemanticMsg+ from openviking.storage.queuefs import SemanticMsg, get_queue_manager- viking_fs = get_viking_fs()queue_manager = get_queue_manager()# Get semantic queue- semantic_queue = queue_manager.get_queue(- queue_manager.SEMANTIC,- allow_create=True- )+ semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)# Sort by depth (descending) for bottom-up processingmsg = SemanticMsg(@@ -251,8 +244,8 @@ class TreeBuilder:if isinstance(result, str):return resultelif isinstance(result, bytes):- return result.decode('utf-8')- elif hasattr(result, 'to_dict') and not isinstance(result, list):+ return result.decode("utf-8")+ elif hasattr(result, "to_dict") and not isinstance(result, list):# Handle FindResult by converting to dict (skip lists)return str(result.to_dict())elif isinstance(result, list):@@ -260,5 +253,3 @@ class TreeBuilder:return json.dumps(result)else:return str(result)--diff --git a/openviking/parse/vlm.py b/openviking/parse/vlm.pyindex 9eade9899..c410bea89 100644--- a/openviking/parse/vlm.py+++ b/openviking/parse/vlm.py@@ -4,9 +4,10 @@from dataclasses import dataclassfrom pathlib import Path-from typing import Any, Dict, List, Optional, Union-from openviking.utils.extractor import ImageInfo, TableInfo+from typing import Any, Dict, List, Union+from openviking.prompts import render_prompt+from openviking.utils.extractor import ImageInfo, TableInfofrom openviking.utils.llm import parse_json_from_responsefrom openviking.utils.logger import get_logger@@ -52,6 +53,7 @@ class VLMProcessor:def _get_vlm(self):"""Get VLM singleton."""from openviking.utils.config import get_openviking_config+return get_openviking_config().vlmasync def understand_image(@@ -125,7 +127,6 @@ class VLMProcessor:# Add sample dataif len(raw_data) > 1:- sample_rows = min(3, len(raw_data) - 1)overview_parts.append(f"Contains {len(raw_data) - 1} data records.")overview = " ".join(overview_parts)@@ -247,7 +248,9 @@ class VLMProcessor:async def process_batch(batch_start: int, batch: List) -> tuple:async with semaphore:- results = await self._batch_understand_pages_single_call(batch, instruction, batch_start)+ results = await self._batch_understand_pages_single_call(+ batch, instruction, batch_start+ )return batch_start, results# Run all batches concurrently@@ -282,7 +285,7 @@ class VLMProcessor:data = parse_json_from_response(response)if not data or "pages" not in data:- raise ValueError(f"Invalid VLM response: missing 'pages' field")+ raise ValueError("Invalid VLM response: missing 'pages' field")results = []for page_info in data["pages"]:@@ -350,7 +353,7 @@ class VLMProcessor:images_section = ""if images:images_section = "\n".join(- f"Image {i+1}: Located on page {img.page + 1}"+ f"Image {i + 1}: Located on page {img.page + 1}"for i, img in enumerate(images[: self.max_images_per_call]))else:@@ -360,7 +363,7 @@ class VLMProcessor:tables_section = ""if tables:tables_section = "\n".join(- f"Table {i+1}: Located on page {tbl.page + 1}"+ f"Table {i + 1}: Located on page {tbl.page + 1}"for i, tbl in enumerate(tables[: self.max_images_per_call]))else:@@ -370,7 +373,7 @@ class VLMProcessor:sections_list = ""if sections:sections_list = "\n".join(- f"Section {i+1}: {sec.get('title', 'Untitled')}"+ f"Section {i + 1}: {sec.get('title', 'Untitled')}"for i, sec in enumerate(sections[: self.max_sections_per_call]))else:@@ -382,7 +385,9 @@ class VLMProcessor:"title": title or "Unknown document","instruction": instruction or "Understand document content","reason": reason or "User added",- "content_preview": content_preview[:2000] if content_preview else "No content preview",+ "content_preview": content_preview[:2000]+ if content_preview+ else "No content preview","image_count": len(images),"images_section": images_section,"table_count": len(tables),@@ -442,7 +447,9 @@ class VLMProcessor:if table.has_structured_data():result = self._understand_table_from_data(table, instruction)else:- result = VLMResult("[Table]", "Cannot parse table (VLM not available)", "[Table content]")+ result = VLMResult(+ "[Table]", "Cannot parse table (VLM not available)", "[Table content]"+ )table_results.append(result)# Simplified prompt for text-only analysis@@ -456,7 +463,7 @@ Content preview:{content_preview[:3000]}Section list:-{chr(10).join(f"- {s.get('title', 'Untitled')}" for s in sections[:self.max_sections_per_call])}+{chr(10).join(f"- {s.get('title', 'Untitled')}" for s in sections[: self.max_sections_per_call])}Please output in JSON format:{{@@ -525,7 +532,7 @@ Please output in JSON format:# Parse image resultsimage_results = []image_data = data.get("images", [])- for i, img in enumerate(images):+ for i, _ in enumerate(images):if i < len(image_data):img_info = image_data[i]result = VLMResult(@@ -540,7 +547,7 @@ Please output in JSON format:# Parse table resultstable_results = []table_data = data.get("tables", [])- for i, tbl in enumerate(tables):+ for i, _ in enumerate(tables):if i < len(table_data):tbl_info = table_data[i]result = VLMResult(@@ -607,11 +614,11 @@ Please output in JSON format:images_info = []image_data_list = []- for i, (img_data, context) in enumerate(batch):+ for i, (_img_data, context) in enumerate(batch):images_info.append(f"Image {i + 1}: Surrounding text: {context[:100] if context else 'No context'}")- image_data_list.append(img_data)+ image_data_list.append(_img_data)prompt = render_prompt("vision.batch_filtering",@@ -631,7 +638,7 @@ Please output in JSON format:data = parse_json_from_response(response)if data and "results" in data:batch_results = []- for i, (img_data, context) in enumerate(batch):+ for i, (_img_data, _context) in enumerate(batch):if i < len(data["results"]):result = data["results"][i]batch_results.append(@@ -656,7 +663,10 @@ Please output in JSON format:logger.error(f"Error filtering image batch: {e}")# On error, keep all images by default- return [{"is_meaningful": True, "reason": "Filtering failed", "image_type": "Unknown"} for _ in batch]+ return [+ {"is_meaningful": True, "reason": "Filtering failed", "image_type": "Unknown"}+ for _ in batch+ ]async def _filter_single_image(self,diff --git a/openviking/retrieve/hierarchical_retriever.py b/openviking/retrieve/hierarchical_retriever.pyindex 4d76e214a..6881e5691 100644--- a/openviking/retrieve/hierarchical_retriever.py+++ b/openviking/retrieve/hierarchical_retriever.py@@ -8,40 +8,37 @@ and rerank-based relevance scoring."""import heapq-import re-from typing import Any, Callable, Dict, List, Optional, Tuple+from typing import Any, Dict, List, Optional, Tuple-from openviking.utils.config import RerankConfig+from openviking.models.embedder.base import EmbedResultfrom openviking.retrieve.types import (ContextType,MatchedContext,QueryResult,RelatedContext,- ScoreDistribution,- ThinkingTrace,- TraceEventType,TypedQuery,)from openviking.storage import VikingDBInterfacefrom openviking.storage.viking_fs import get_viking_fs-from openviking.models.embedder.base import EmbedResult+from openviking.utils.config import RerankConfigfrom openviking.utils.logger import get_logger-from openviking.utils.rerank import RerankClientlogger = get_logger(__name__)+class RetrieverMode(str):THINKING = "thinking"QUICK = "quick"+class HierarchicalRetriever:"""Hierarchical retriever with dense and sparse vector support."""- MAX_CONVERGENCE_ROUNDS = 3 # Stop after multiple rounds with unchanged topk- MAX_RELATIONS = 5 # Maximum relations per resource- SCORE_PROPAGATION_ALPHA = 0.5 # Score propagation coefficient- DIRECTORY_DOMINANCE_RATIO = 1.2 # Directory score must exceed max child score- GLOBAL_SEARCH_TOPK = 3 # Global retrieval count+ MAX_CONVERGENCE_ROUNDS = 3 # Stop after multiple rounds with unchanged topk+ MAX_RELATIONS = 5 # Maximum relations per resource+ SCORE_PROPAGATION_ALPHA = 0.5 # Score propagation coefficient+ DIRECTORY_DOMINANCE_RATIO = 1.2 # Directory score must exceed max child score+ GLOBAL_SEARCH_TOPK = 3 # Global retrieval countdef __init__(self,@@ -67,10 +64,14 @@ class HierarchicalRetriever:if rerank_config and rerank_config.is_available():# TODO: Support later - initialize RerankClient hereself._rerank_client = None- logger.info(f"[HierarchicalRetriever] Rerank config available, threshold={self.threshold}")+ logger.info(+ f"[HierarchicalRetriever] Rerank config available, threshold={self.threshold}"+ )else:self._rerank_client = None- logger.info(f"[HierarchicalRetriever] Rerank not configured, using vector search only with threshold={self.threshold}")+ logger.info(+ f"[HierarchicalRetriever] Rerank not configured, using vector search only with threshold={self.threshold}"+ )async def retrieve(self,@@ -105,10 +106,7 @@ class HierarchicalRetriever:if metadata_filter:filters_to_merge.append(metadata_filter)- final_metadata_filter = {- "op": "and",- "conds": filters_to_merge- }+ final_metadata_filter = {"op": "and", "conds": filters_to_merge}# Step 1: Determine starting directories based on context_typeroot_uris = self._get_root_uris_for_type(query.context_type)@@ -163,13 +161,16 @@ class HierarchicalRetriever:return []sparse_query_vector = result.sparse_vector or {}- global_filter = {"op": "and", "conds": [filter, {"op": "must", "field": "is_leaf", "conds": [False]}]}+ global_filter = {+ "op": "and",+ "conds": [filter, {"op": "must", "field": "is_leaf", "conds": [False]}],+ }results = await self.storage.search(collection=collection,query_vector=query_vector,sparse_query_vector=sparse_query_vector,filter=global_filter,- limit=limit+ limit=limit,)return results@@ -192,7 +193,7 @@ class HierarchicalRetriever:if self._rerank_client and mode == RetrieverMode.THINKING:for r in global_results:# todo: multi-modal- doc = r['abstract']+ doc = r["abstract"]docs.append(doc)rerank_scores = self._rerank_client.rerank_batch(query, docs)for i, r in enumerate(global_results):@@ -244,10 +245,7 @@ class HierarchicalRetriever:"""Merge filter conditions."""if not extra_filter:return base_filter- return {- "op": "and",- "conds": [base_filter, extra_filter]- }+ return {"op": "and", "conds": [base_filter, extra_filter]}# Generate query vectorsquery_vector = None@@ -288,7 +286,9 @@ class HierarchicalRetriever:collection=collection,query_vector=query_vector,sparse_query_vector=sparse_query_vector, # Pass sparse vector- filter=merge_filter({'op': 'must', "field": "parent_uri", "conds":[current_uri]}, metadata_filter),+ filter=merge_filter(+ {"op": "must", "field": "parent_uri", "conds": [current_uri]}, metadata_filter+ ),limit=pre_filter_limit,)@@ -300,7 +300,7 @@ class HierarchicalRetriever:documents = []for r in results:# todo: multi-modal- doc = r['abstract']+ doc = r["abstract"]documents.append(doc)rerank_scores = self._rerank_client.rerank_batch(query, documents)@@ -311,23 +311,29 @@ class HierarchicalRetriever:for r, score in zip(results, query_scores):uri = r.get("uri", "")- final_score = alpha * score + (1 - alpha) * current_score if current_score else score+ final_score = (+ alpha * score + (1 - alpha) * current_score if current_score else score+ )if passes_threshold(final_score) and uri not in visited:r["_final_score"] = final_scorecollected.append(r)visited.add(uri)- logger.info(f"[RecursiveSearch] Added URI: {uri} to candidates with score: {final_score}")+ logger.info(+ f"[RecursiveSearch] Added URI: {uri} to candidates with score: {final_score}"+ )if not r.get("is_leaf"):continueheapq.heappush(dir_queue, (-final_score, uri))else:- logger.info(f"[RecursiveSearch] URI {uri} score {final_score} did not pass threshold {effective_threshold}")+ logger.info(+ f"[RecursiveSearch] URI {uri} score {final_score} did not pass threshold {effective_threshold}"+ )# Convergence check- current_topk = sorted(- collected, key=lambda x: x.get("_final_score", 0), reverse=True- )[:limit]+ current_topk = sorted(collected, key=lambda x: x.get("_final_score", 0), reverse=True)[+ :limit+ ]current_topk_uris = {c.get("uri", "") for c in current_topk}if current_topk_uris == prev_topk_uris and len(current_topk_uris) >= limit:diff --git a/openviking/retrieve/intent_analyzer.py b/openviking/retrieve/intent_analyzer.pyindex 1a6cb95cb..0d5b4725f 100644--- a/openviking/retrieve/intent_analyzer.py+++ b/openviking/retrieve/intent_analyzer.py@@ -6,12 +6,12 @@ Intent analyzer for OpenViking retrieval.Analyzes session context to generate query plans."""-from typing import Any, Dict, List, Optional+from typing import List, Optional-from openviking.utils.config import get_openviking_configfrom openviking.message import Messagefrom openviking.prompts import render_promptfrom openviking.retrieve.types import ContextType, QueryPlan, TypedQuery+from openviking.utils.config import get_openviking_configfrom openviking.utils.llm import parse_json_from_responsefrom openviking.utils.logger import get_logger@@ -59,9 +59,7 @@ class IntentAnalyzer:)# Call LLM- response = await get_openviking_config().vlm.get_completion_async(- prompt- )+ response = await get_openviking_config().vlm.get_completion_async(prompt)# Parse resultparsed = parse_json_from_response(response)@@ -88,7 +86,7 @@ class IntentAnalyzer:# Log analysis resultfor i, q in enumerate(queries):logger.info(- f' [{i+1}] type={q.context_type.value}, priority={q.priority}, query="{q.query}"'+ f' [{i + 1}] type={q.context_type.value}, priority={q.priority}, query="{q.query}"')logger.debug(f"[IntentAnalyzer] Reasoning: {parsed.get('reasoning', '')[:200]}...")@@ -111,11 +109,9 @@ class IntentAnalyzer:summary = compression_summary if compression_summary else "None"# Format recent messages- recent = messages[-self.max_recent_messages:] if messages else []+ recent = messages[-self.max_recent_messages :] if messages else []recent_messages = (- "\n".join(f"[{m.role}]: {m.content}" for m in recent if m.content)- if recent- else "None"+ "\n".join(f"[{m.role}]: {m.content}" for m in recent if m.content) if recent else "None")# Current messagediff --git a/openviking/retrieve/types.py b/openviking/retrieve/types.pyindex a38eef660..04a50e4b7 100644--- a/openviking/retrieve/types.py+++ b/openviking/retrieve/types.py@@ -272,6 +272,7 @@ class QueryPlan:@dataclassclass RelatedContext:"""Related context with summary."""+uri: strabstract: strdiff --git a/openviking/session/compressor.py b/openviking/session/compressor.pyindex e36574a3a..8c4981dc8 100644--- a/openviking/session/compressor.py+++ b/openviking/session/compressor.py@@ -8,11 +8,10 @@ Uses MemoryExtractor for 6-category extraction and MemoryDeduplicator for LLM-ba"""from dataclasses import dataclass-from typing import Any, Dict, List, Optional+from typing import Dict, Listfrom openviking.core.context import Contextfrom openviking.message import Message-from openviking.utils.config import get_openviking_configfrom openviking.storage import VikingDBManagerfrom openviking.storage.viking_fs import get_viking_fsfrom openviking.utils import get_logger@@ -48,6 +47,7 @@ class SessionCompressor:async def _index_memory(self, memory: Context) -> bool:"""Add memory to vectorization queue."""from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter+embedding_msg = EmbeddingMsgConverter.from_context(memory)await self.vikingdb.enqueue_embedding_msg(embedding_msg)logger.info(f"Enqueued memory for vectorization: {memory.uri}")@@ -149,16 +149,22 @@ class SessionCompressor:for memory_uri in memory_uris:if resource_uris:await viking_fs.link(- memory_uri, resource_uris, reason="Memory extracted from session using these resources"+ memory_uri,+ resource_uris,+ reason="Memory extracted from session using these resources",)if skill_uris:await viking_fs.link(- memory_uri, skill_uris, reason="Memory extracted from session calling these skills"+ memory_uri,+ skill_uris,+ reason="Memory extracted from session calling these skills",)# Resources/skills -> memories (reverse)for resource_uri in resource_uris:- await viking_fs.link(resource_uri, memory_uris, reason="Referenced by these memories")+ await viking_fs.link(+ resource_uri, memory_uris, reason="Referenced by these memories"+ )for skill_uri in skill_uris:await viking_fs.link(skill_uri, memory_uris, reason="Called by these memories")diff --git a/openviking/session/memory_deduplicator.py b/openviking/session/memory_deduplicator.pyindex 64dad6cf3..266efda68 100644--- a/openviking/session/memory_deduplicator.py+++ b/openviking/session/memory_deduplicator.py@@ -8,16 +8,16 @@ LLM-assisted deduplication with CREATE/UPDATE/MERGE/SKIP decisions (Mem0 inspirefrom dataclasses import dataclassfrom enum import Enum-from typing import Any, Dict, List, Optional+from typing import List, Optional-from openviking.utils.config import get_openviking_config-from openviking.core.context import Context, ContextType+from openviking.core.context import Context+from openviking.models.embedder.base import EmbedResultfrom openviking.prompts import render_promptfrom openviking.storage import VikingDBManagerfrom openviking.utils import get_logger-from openviking.models.embedder.base import EmbedResult+from openviking.utils.config import get_openviking_config-from .memory_extractor import CandidateMemory, MemoryCategory+from .memory_extractor import CandidateMemorylogger = get_logger(__name__)diff --git a/openviking/session/memory_extractor.py b/openviking/session/memory_extractor.pyindex bb23fe0c8..6c664af19 100644--- a/openviking/session/memory_extractor.py+++ b/openviking/session/memory_extractor.py@@ -9,17 +9,15 @@ Extracts 6 categories of memories from session:"""from dataclasses import dataclass-from datetime import datetimefrom enum import Enum-from typing import Any, List, Optional+from typing import List, Optionalfrom uuid import uuid4-from openviking.utils.config import get_openviking_configfrom openviking.core.context import Context, ContextType, Vectorize-from openviking.message import Messagefrom openviking.prompts import render_promptfrom openviking.storage.viking_fs import get_viking_fsfrom openviking.utils import get_logger+from openviking.utils.config import get_openviking_configlogger = get_logger(__name__)@@ -82,11 +80,7 @@ class MemoryExtractor:# Format all messagesmessages = context["messages"]- formatted_messages = "\n".join([- f"[{m.role}]: {m.content}"- for m in messages- if m.content- ])+ formatted_messages = "\n".join([f"[{m.role}]: {m.content}" for m in messages if m.content])if not formatted_messages:return []diff --git a/openviking/session/session.py b/openviking/session/session.pyindex 46bd36a6f..764e40c42 100644--- a/openviking/session/session.py+++ b/openviking/session/session.py@@ -13,9 +13,9 @@ from datetime import datetimefrom typing import TYPE_CHECKING, Any, Dict, List, Optionalfrom uuid import uuid4-from openviking.utils.config import get_openviking_configfrom openviking.message import Message, Partfrom openviking.utils import get_logger+from openviking.utils.config import get_openviking_configif TYPE_CHECKING:from openviking.sync_client import OpenViking@@ -29,6 +29,7 @@ def _run_async(coro):loop = asyncio.get_running_loop()# If already in event loop, use nest_asyncio or return directlyimport nest_asyncio+nest_asyncio.apply()return loop.run_until_complete(coro)except RuntimeError:@@ -102,9 +103,9 @@ class Session:returntry:- content = _run_async(self.client.viking_fs.read_file(- f"{self._session_uri}/messages.jsonl"- ))+ content = _run_async(+ self.client.viking_fs.read_file(f"{self._session_uri}/messages.jsonl")+ )self._messages = [Message.from_dict(json.loads(line))for line in content.strip().split("\n")@@ -117,7 +118,9 @@ class Session:# Restore compression_index (scan history directory)try:history_items = _run_async(self.client.viking_fs.ls(f"{self._session_uri}/history"))- archives = [item["name"] for item in history_items if item["name"].startswith("archive_")]+ archives = [+ item["name"] for item in history_items if item["name"].startswith("archive_")+ ]if archives:max_index = max(int(a.split("_")[1]) for a in archives)self._compression.compression_index = max_index@@ -245,11 +248,13 @@ class Session:logger.info(f"Starting memory extraction from {len(messages_to_archive)} archived messages")- memories = _run_async(self.client._session_compressor.extract_long_term_memories(- messages=messages_to_archive,- user=self.user,- session_id=self.session_id,- ))+ memories = _run_async(+ self.client._session_compressor.extract_long_term_memories(+ messages=messages_to_archive,+ user=self.user,+ session_id=self.session_id,+ )+ )logger.info(f"Extracted {len(memories)} memories")result["memories_extracted"] = len(memories)self._stats.memories_extracted += len(memories)@@ -287,11 +292,13 @@ class Session:for usage in self._usage_records:try:- _run_async(storage.update(- collection="context",- filter={"uri": usage.uri},- update={"$inc": {"active_count": 1}},- ))+ _run_async(+ storage.update(+ collection="context",+ filter={"uri": usage.uri},+ update={"$inc": {"active_count": 1}},+ )+ )updated += 1except Exception as e:logger.debug(f"Could not update active_count for {usage.uri}: {e}")@@ -321,9 +328,7 @@ class Session:summaries = []if self.compression.compression_index > 0:try:- history_items = _run_async(self.client.viking_fs.ls(- f"{self._session_uri}/history"- ))+ history_items = _run_async(self.client.viking_fs.ls(f"{self._session_uri}/history"))query_lower = query.lower()# Collect all archives with relevance scores@@ -409,10 +414,12 @@ class Session:# Write messages.jsonllines = [m.to_jsonl() for m in messages]- _run_async(viking_fs.write_file(- uri=f"{archive_uri}/messages.jsonl",- content="\n".join(lines) + "\n",- ))+ _run_async(+ viking_fs.write_file(+ uri=f"{archive_uri}/messages.jsonl",+ content="\n".join(lines) + "\n",+ )+ )_run_async(viking_fs.write_file(uri=f"{archive_uri}/.abstract.md", content=abstract))_run_async(viking_fs.write_file(uri=f"{archive_uri}/.overview.md", content=overview))@@ -433,29 +440,37 @@ class Session:lines = [m.to_jsonl() for m in messages]content = "\n".join(lines) + "\n" if lines else ""- _run_async(viking_fs.write_file(- uri=f"{self._session_uri}/messages.jsonl",- content=content,- ))+ _run_async(+ viking_fs.write_file(+ uri=f"{self._session_uri}/messages.jsonl",+ content=content,+ )+ )# Update L0/L1- _run_async(viking_fs.write_file(- uri=f"{self._session_uri}/.abstract.md",- content=abstract,- ))- _run_async(viking_fs.write_file(- uri=f"{self._session_uri}/.overview.md",- content=overview,- ))+ _run_async(+ viking_fs.write_file(+ uri=f"{self._session_uri}/.abstract.md",+ content=abstract,+ )+ )+ _run_async(+ viking_fs.write_file(+ uri=f"{self._session_uri}/.overview.md",+ content=overview,+ )+ )def _append_to_jsonl(self, msg: Message) -> None:"""Append to messages.jsonl."""if not self.client or not self.client.viking_fs:return- _run_async(self.client.viking_fs.append_file(- f"{self._session_uri}/messages.jsonl",- msg.to_jsonl() + "\n",- ))+ _run_async(+ self.client.viking_fs.append_file(+ f"{self._session_uri}/messages.jsonl",+ msg.to_jsonl() + "\n",+ )+ )def _update_message_in_jsonl(self) -> None:"""Update message in messages.jsonl."""@@ -464,10 +479,12 @@ class Session:lines = [m.to_jsonl() for m in self._messages]content = "\n".join(lines) + "\n"- _run_async(self.client.viking_fs.write_file(- f"{self._session_uri}/messages.jsonl",- content,- ))+ _run_async(+ self.client.viking_fs.write_file(+ f"{self._session_uri}/messages.jsonl",+ content,+ )+ )def _save_tool_result(self,@@ -493,10 +510,12 @@ class Session:"status": status,"time": {"created": datetime.now().isoformat()},}- _run_async(self.client.viking_fs.write_file(- f"{self._session_uri}/tools/{tool_id}/tool.json",- json.dumps(tool_data, ensure_ascii=False),- ))+ _run_async(+ self.client.viking_fs.write_file(+ f"{self._session_uri}/tools/{tool_id}/tool.json",+ json.dumps(tool_data, ensure_ascii=False),+ )+ )def _generate_abstract(self) -> str:"""Generate one-sentence summary for session."""@@ -516,7 +535,9 @@ class Session:f"- `messages.jsonl` - Current messages ({turn_count} turns)",]if self._compression.compression_index > 0:- parts.append(f"- `history/` - Historical archives ({self._compression.compression_index} total)")+ parts.append(+ f"- `history/` - Historical archives ({self._compression.compression_index} total)"+ )parts.extend(["",diff --git a/openviking/storage/__init__.py b/openviking/storage/__init__.pyindex b3cffd909..636207224 100644--- a/openviking/storage/__init__.py+++ b/openviking/storage/__init__.py@@ -2,6 +2,10 @@# SPDX-License-Identifier: Apache-2.0"""Storage layer interfaces and implementations."""+from openviking.storage.observers import BaseObserver, QueueObserver+from openviking.storage.queuefs import QueueManager, get_queue_manager, init_queue_manager+from openviking.storage.viking_fs import VikingFS, get_viking_fs, init_viking_fs+from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackendfrom openviking.storage.vikingdb_interface import (CollectionNotFoundError,ConnectionError,@@ -11,16 +15,7 @@ from openviking.storage.vikingdb_interface import (StorageException,VikingDBInterface,)--from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackendfrom openviking.storage.vikingdb_manager import VikingDBManager-from openviking.storage.queuefs import QueueManager, get_queue_manager, init_queue_manager-from openviking.storage.viking_fs import (- VikingFS,- get_viking_fs,- init_viking_fs,-)-from openviking.storage.observers import BaseObserver, QueueObserver__all__ = [# Interfacediff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex 7e1ab2905..6b51fb929 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -7,20 +7,21 @@ Provides centralized schema definitions and factory functions for creating collesimilar to how init_viking_fs encapsulates VikingFS initialization."""-import abcimport json-from typing import Dict, Any, Optional-from openviking.storage.queuefs.named_queue import DequeueHandlerBase+from typing import Any, Dict, Optional++from openviking.models.embedder.base import EmbedResultfrom openviking.storage.queuefs.embedding_msg import EmbeddingMsg+from openviking.storage.queuefs.named_queue import DequeueHandlerBasefrom openviking.storage.vikingdb_interface import VikingDBInterfacefrom openviking.utils import get_loggerfrom openviking.utils.config.open_viking_config import OpenVikingConfig-from openviking.models.embedder.base import EmbedResultlogger = get_logger(__name__)COLLECTION_NAME = "context"+class CollectionSchemas:"""Centralized collection schema definitions.@@ -72,12 +73,14 @@ async def init_context_collection(storage) -> bool:True if collection was created, False if already exists"""from openviking.utils.config import get_openviking_config+config = get_openviking_config()name = config.storage.vectordb.namevector_dim = config.embedding.dimensionschema = CollectionSchemas.context_collection(name, vector_dim)return await storage.create_collection(name, schema)+class TextEmbeddingHandler(DequeueHandlerBase):"""Text embedding handler that converts text messages to embedding vectors@@ -97,6 +100,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):vikingdb: VikingDBInterface instance for writing to vector database"""from openviking.utils.config import get_openviking_config+self._vikingdb = vikingdbself._embedder = Noneconfig = get_openviking_config()@@ -128,6 +132,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):# Initialize embedder if not already initializedif not self._embedder:from openviking.utils.config import get_openviking_config+config = get_openviking_config()self._initialize_embedder(config)@@ -159,10 +164,13 @@ class TextEmbeddingHandler(DequeueHandlerBase):try:record_id = await self._vikingdb.insert(self._collection_name, inserted_data)if record_id:- logger.debug(f"Successfully wrote embedding to database: {record_id} abstract {inserted_data['abstract']} vector {inserted_data['vector'][:5]}")+ logger.debug(+ f"Successfully wrote embedding to database: {record_id} abstract {inserted_data['abstract']} vector {inserted_data['vector'][:5]}"+ )except Exception as db_err:logger.error(f"Failed to write to vector database: {db_err}")import traceback+traceback.print_exc()self.report_error(str(db_err), data)return None@@ -173,6 +181,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):except Exception as e:logger.error(f"Error processing embedding message: {e}")import traceback+traceback.print_exc()self.report_error(str(e), data)return Nonediff --git a/openviking/storage/observers/vlm_observer.py b/openviking/storage/observers/vlm_observer.pyindex 28dc709e5..561c98823 100644--- a/openviking/storage/observers/vlm_observer.py+++ b/openviking/storage/observers/vlm_observer.py@@ -6,8 +6,8 @@ VLMObserver: VLM system observability tool.Provides methods to observe and report token usage across VLM models and backends."""-from openviking.storage.observers.base_observer import BaseObserverfrom openviking.models.vlm.base import VLMBase+from openviking.storage.observers.base_observer import BaseObserverfrom openviking.utils.logger import get_loggerlogger = get_logger(__name__)diff --git a/openviking/storage/queuefs/__init__.py b/openviking/storage/queuefs/__init__.pyindex f9e19339e..7016da0ec 100644--- a/openviking/storage/queuefs/__init__.py+++ b/openviking/storage/queuefs/__init__.py@@ -2,11 +2,11 @@# SPDX-License-Identifier: Apache-2.0from .embedding_msg import EmbeddingMsgfrom .embedding_queue import EmbeddingQueue+from .named_queue import NamedQueue, QueueError, QueueStatus+from .queue_manager import QueueManager, get_queue_manager, init_queue_managerfrom .semantic_msg import SemanticMsg-from .semantic_queue import SemanticQueuefrom .semantic_processor import SemanticProcessor-from .named_queue import NamedQueue, QueueStatus, QueueError-from .queue_manager import QueueManager, get_queue_manager, init_queue_manager+from .semantic_queue import SemanticQueue__all__ = ["QueueManager",diff --git a/openviking/storage/queuefs/embedding_msg.py b/openviking/storage/queuefs/embedding_msg.pyindex e599a8445..19b8381ea 100644--- a/openviking/storage/queuefs/embedding_msg.py+++ b/openviking/storage/queuefs/embedding_msg.py@@ -1,9 +1,9 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0import json+from dataclasses import asdict, dataclass+from typing import Any, Dict, List, Unionfrom uuid import uuid4-from dataclasses import dataclass, asdict-from typing import List, Dict, Any, Optional, Union@dataclass@@ -11,9 +11,7 @@ class EmbeddingMsg:message: Union[str, List[Dict[str, Any]]]context_data: Dict[str, Any]- def __init__(- self, message: Union[str, List[Dict[str, Any]]], context_data: Dict[str, Any]- ):+ def __init__(self, message: Union[str, List[Dict[str, Any]]], context_data: Dict[str, Any]):self.id = str(uuid4())self.message = messageself.context_data = context_data@@ -21,11 +19,11 @@ class EmbeddingMsg:def to_dict(self) -> Dict[str, Any]:"""Convert embedding message to dictionary format."""return asdict(self)-+def to_json(self) -> str:"""Convert embedding message to JSON string."""return json.dumps(self.to_dict())-+@classmethoddef from_dict(cls, data: Dict[str, Any]) -> "EmbeddingMsg":"""Create an embedding message object from dictionary."""@@ -43,4 +41,4 @@ class EmbeddingMsg:data = json.loads(json_str)return cls.from_dict(data)except json.JSONDecodeError as e:- raise ValueError(f"Invalid JSON string: {e}")\ No newline at end of file+ raise ValueError(f"Invalid JSON string: {e}")diff --git a/openviking/storage/queuefs/embedding_msg_converter.py b/openviking/storage/queuefs/embedding_msg_converter.pyindex 577257fe3..3c70ec780 100644--- a/openviking/storage/queuefs/embedding_msg_converter.py+++ b/openviking/storage/queuefs/embedding_msg_converter.py@@ -7,8 +7,6 @@ This module provides a unified interface for converting Context objectsto EmbeddingMsg objects for asynchronous vector processing."""-import json-from openviking.core.context import Contextfrom openviking.storage.queuefs.embedding_msg import EmbeddingMsgfrom openviking.utils import get_logger@@ -36,5 +34,3 @@ class EmbeddingMsgConverter:if hasattr(embedding_msg.context_data, key) and value is not None:setattr(embedding_msg.context_data, key, value)return embedding_msg--diff --git a/openviking/storage/queuefs/embedding_queue.py b/openviking/storage/queuefs/embedding_queue.pyindex 06f23e5a8..57ad0e892 100644--- a/openviking/storage/queuefs/embedding_queue.py+++ b/openviking/storage/queuefs/embedding_queue.py@@ -1,15 +1,15 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0-import json-from dataclasses import asdict-from typing import Any, Dict, Optional, Union+from typing import Optional-from .named_queue import NamedQueue-from .embedding_msg import EmbeddingMsgfrom openviking.utils.logger import get_logger+from .embedding_msg import EmbeddingMsg+from .named_queue import NamedQueue+logger = get_logger(__name__)+class EmbeddingQueue(NamedQueue):"""EmbeddingQueue: Named queue specifically for processing EmbeddingMsg.@@ -43,13 +43,13 @@ class EmbeddingQueue(NamedQueue):data_dict = await super().peek()if not data_dict:return None-+if "data" in data_dict and isinstance(data_dict["data"], str):try:return EmbeddingMsg.from_json(data_dict["data"])except Exception:return None-+try:return EmbeddingMsg.from_dict(data_dict)except Exception:diff --git a/openviking/storage/queuefs/named_queue.py b/openviking/storage/queuefs/named_queue.pyindex ee0b7eca4..046e9c76b 100644--- a/openviking/storage/queuefs/named_queue.py+++ b/openviking/storage/queuefs/named_queue.py@@ -6,7 +6,7 @@ import jsonimport threadingfrom dataclasses import dataclass, fieldfrom datetime import datetime-from typing import Any, Dict, List, Optional, Union, Callable, TYPE_CHECKING+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Unionfrom openviking.utils.logger import get_logger@@ -19,6 +19,7 @@ logger = get_logger(__name__)@dataclassclass QueueError:"""Error record."""+timestamp: datetimemessage: strdata: Optional[Dict[str, Any]] = None@@ -27,6 +28,7 @@ class QueueError:@dataclassclass QueueStatus:"""Queue status."""+pending: int = 0in_progress: int = 0processed: int = 0@@ -138,13 +140,15 @@ class NamedQueue:with self._lock:self._in_progress -= 1self._error_count += 1- self._errors.append(QueueError(- timestamp=datetime.now(),- message=error_msg,- data=data,- ))+ self._errors.append(+ QueueError(+ timestamp=datetime.now(),+ message=error_msg,+ data=data,+ )+ )if len(self._errors) > self.MAX_ERRORS:- self._errors = self._errors[-self.MAX_ERRORS:]+ self._errors = self._errors[-self.MAX_ERRORS :]async def get_status(self) -> QueueStatus:"""Get queue status."""@@ -194,7 +198,9 @@ class NamedQueue:data = json.dumps(data)loop = asyncio.get_event_loop()- msg_id = await loop.run_in_executor(None, self._agfs.write, enqueue_file, data.encode("utf-8"))+ msg_id = await loop.run_in_executor(+ None, self._agfs.write, enqueue_file, data.encode("utf-8")+ )return msg_id if isinstance(msg_id, str) else str(msg_id)async def dequeue(self) -> Optional[Dict[str, Any]]:@@ -223,7 +229,7 @@ class NamedQueue:"""Peek at head message without removing."""await self._ensure_initialized()peek_file = f"{self.path}/peek"-+loop = asyncio.get_event_loop()try:content = await loop.run_in_executor(None, self._agfs.read, peek_file)@@ -238,7 +244,7 @@ class NamedQueue:"""Get queue size."""await self._ensure_initialized()size_file = f"{self.path}/size"-+loop = asyncio.get_event_loop()try:content = await loop.run_in_executor(None, self._agfs.read, size_file)@@ -253,7 +259,7 @@ class NamedQueue:"""Clear queue."""await self._ensure_initialized()clear_file = f"{self.path}/clear"-+loop = asyncio.get_event_loop()try:await loop.run_in_executor(None, self._agfs.write, clear_file, b"")diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.pyindex 6f40df455..7d3a76ff8 100644--- a/openviking/storage/queuefs/queue_manager.py+++ b/openviking/storage/queuefs/queue_manager.py@@ -5,15 +5,16 @@ QueueManager: Encapsulates AGFS QueueFS plugin operations.All queues are managed through NamedQueue."""-import timeimport asyncioimport atexitimport threading-from typing import Any, Dict, List, Optional, Union+import time+from typing import Any, Dict, Optional, Unionfrom openviking.utils.logger import get_logger-from .named_queue import NamedQueue, EnqueueHookBase, DequeueHandlerBase, QueueStatus+from .embedding_queue import EmbeddingQueue+from .named_queue import DequeueHandlerBase, EnqueueHookBase, NamedQueue, QueueStatusfrom .semantic_queue import SemanticQueuelogger = get_logger(__name__)diff --git a/openviking/storage/queuefs/semantic_msg.py b/openviking/storage/queuefs/semantic_msg.pyindex 899e215f1..e4d30be82 100644--- a/openviking/storage/queuefs/semantic_msg.py+++ b/openviking/storage/queuefs/semantic_msg.py@@ -3,15 +3,16 @@"""SemanticMsg: Semantic extraction queue message dataclass."""import json-from uuid import uuid4-from dataclasses import dataclass, asdict, field-from typing import List, Dict, Any, Optional+from dataclasses import asdict, dataclassfrom datetime import datetime+from typing import Any, Dict+from uuid import uuid4@dataclassclass SemanticMsg:"""Semantic extraction queue message."""+id: str # UUIDuri: str # Directory URIcontext_type: str # resource, memory, skilldiff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex dcaabfc67..7e300666a 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -3,16 +3,16 @@"""SemanticProcessor: Processes messages from SemanticQueue, generates .abstract.md and .overview.md."""import asyncio-from typing import List, Dict, Any, Tuple+from typing import Any, Dict, List, Tuple-from openviking.utils import VikingURI-from openviking.utils.logger import get_logger-from openviking.utils.config import get_openviking_config+from openviking.core.context import Context, ResourceContentType, Vectorizefrom openviking.prompts import render_prompt-from openviking.storage.viking_fs import get_viking_fsfrom openviking.storage.queuefs.named_queue import DequeueHandlerBasefrom openviking.storage.queuefs.semantic_msg import SemanticMsg-from openviking.core.context import Context, Vectorize, ResourceContentType+from openviking.storage.viking_fs import get_viking_fs+from openviking.utils import VikingURI+from openviking.utils.config import get_openviking_config+from openviking.utils.logger import get_loggerlogger = get_logger(__name__)@@ -77,6 +77,7 @@ class SemanticProcessor(DequeueHandlerBase):"""Process dequeued SemanticMsg, recursively process all subdirectories."""try:import json+if "data" in data and isinstance(data["data"], str):data = json.loads(data["data"])@@ -146,9 +147,7 @@ class SemanticProcessor(DequeueHandlerBase):except Exception as e:logger.error(f"Failed to vectorize files in {uri}: {e}", exc_info=True)- async def _collect_children_abstracts(- self, children_uris: List[str]- ) -> List[Dict[str, str]]:+ async def _collect_children_abstracts(self, children_uris: List[str]) -> List[Dict[str, str]]:"""Collect .abstract.md from subdirectories."""viking_fs = get_viking_fs()results = []@@ -159,9 +158,7 @@ class SemanticProcessor(DequeueHandlerBase):results.append({"name": dir_name, "abstract": abstract})return results- async def _generate_file_summaries(- self, file_paths: List[str]- ) -> List[Dict[str, str]]:+ async def _generate_file_summaries(self, file_paths: List[str]) -> List[Dict[str, str]]:"""Concurrently generate file summaries."""if not file_paths:return []@@ -218,26 +215,26 @@ class SemanticProcessor(DequeueHandlerBase):def _extract_abstract_from_overview(self, overview_content: str) -> str:"""Extract abstract from overview.md."""- lines = overview_content.split('\n')+ lines = overview_content.split("\n")# Skip header lines (starting with #)content_lines = []in_header = Truefor line in lines:- if in_header and line.startswith('#'):+ if in_header and line.startswith("#"):continueelif in_header and line.strip():in_header = Falseif not in_header:# Stop at first ##- if line.startswith('##'):+ if line.startswith("##"):breakif line.strip():content_lines.append(line.strip())- return '\n'.join(content_lines).strip()+ return "\n".join(content_lines).strip()async def _generate_overview(self,@@ -256,6 +253,7 @@ class SemanticProcessor(DequeueHandlerBase):Overview content"""import re+vlm = get_openviking_config().vlmif not vlm.is_available():@@ -266,14 +264,16 @@ class SemanticProcessor(DequeueHandlerBase):file_index_map = {}file_summaries_lines = []for idx, item in enumerate(file_summaries, 1):- file_index_map[idx] = item['name']+ file_index_map[idx] = item["name"]file_summaries_lines.append(f"[{idx}] {item['name']}: {item['summary']}")file_summaries_str = "\n".join(file_summaries_lines) if file_summaries_lines else "None"# Build subdirectory summary string- children_abstracts_str = "\n".join(- f"- {item['name']}/: {item['abstract']}" for item in children_abstracts- ) if children_abstracts else "None"+ children_abstracts_str = (+ "\n".join(f"- {item['name']}/: {item['abstract']}" for item in children_abstracts)+ if children_abstracts+ else "None"+ )# Generate overviewtry:@@ -293,7 +293,7 @@ class SemanticProcessor(DequeueHandlerBase):idx = int(match.group(1))return file_index_map.get(idx, match.group(0))- overview = re.sub(r'\[(\d+)\]', replace_index, overview)+ overview = re.sub(r"\[(\d+)\]", replace_index, overview)return overview.strip()@@ -305,9 +305,9 @@ class SemanticProcessor(DequeueHandlerBase):self, uri: str, context_type: str, abstract: str, overview: str) -> None:"""Create directory Context and enqueue to EmbeddingQueue."""+from openviking.storage.queuefs import get_queue_managerfrom openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter- from datetime import datetimeparent_uri = VikingURI(uri).parent.uricontext = Context(@@ -315,7 +315,7 @@ class SemanticProcessor(DequeueHandlerBase):parent_uri=parent_uri,is_leaf=False,abstract=abstract,- context_type=context_type+ context_type=context_type,)context.set_vectorize(Vectorize(text=overview))@@ -333,10 +333,11 @@ class SemanticProcessor(DequeueHandlerBase):file_summaries: List[Dict[str, str]],) -> None:"""Vectorize files in directory."""+ from datetime import datetime+from openviking.core.context import Contextfrom openviking.storage.queuefs import get_queue_managerfrom openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter- from datetime import datetimequeue_manager = get_queue_manager()embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)@@ -361,7 +362,7 @@ class SemanticProcessor(DequeueHandlerBase):embedding_msg = EmbeddingMsgConverter.from_context(context)await embedding_queue.enqueue(embedding_msg)logger.debug(f"Enqueued file for vectorization: {file_path}")-+def get_resource_content_type(self, file_name: str) -> ResourceContentType:def _is_image_file(file_name: str) -> bool:image_extensions = {".png", ".jpg", ".jpeg", ".gif", ".bmp", ".svg", ".webp"}@@ -370,11 +371,11 @@ class SemanticProcessor(DequeueHandlerBase):def _is_video_file(file_name: str) -> bool:video_extensions = {".mp4", ".avi", ".mov", ".wmv", ".flv"}return any(file_name.endswith(ext) for ext in video_extensions)-+def _is_text_file(file_name: str) -> bool:text_extensions = {".txt", ".md", ".csv", ".json", ".xml"}return any(file_name.endswith(ext) for ext in text_extensions)-+def _is_audio_file(file_name: str) -> bool:audio_extensions = {".mp3", ".wav", ".aac", ".flac"}return any(file_name.endswith(ext) for ext in audio_extensions)@@ -387,5 +388,5 @@ class SemanticProcessor(DequeueHandlerBase):return ResourceContentType.VIDEOelif _is_audio_file(file_name):return ResourceContentType.AUDIO-+return ResourceContentType.TEXTdiff --git a/openviking/storage/queuefs/semantic_queue.py b/openviking/storage/queuefs/semantic_queue.pyindex feb1edbbc..d758f3bee 100644--- a/openviking/storage/queuefs/semantic_queue.py+++ b/openviking/storage/queuefs/semantic_queue.py@@ -5,6 +5,7 @@from typing import Optionalfrom openviking.utils.logger import get_logger+from .named_queue import NamedQueuefrom .semantic_msg import SemanticMsgdiff --git a/openviking/storage/vectordb/__init__.py b/openviking/storage/vectordb/__init__.pyindex 8ec4a0931..34a1a0f9f 100644--- a/openviking/storage/vectordb/__init__.py+++ b/openviking/storage/vectordb/__init__.py@@ -1,4 +1,3 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0"""VectorDB storage backend."""-diff --git a/openviking/storage/vectordb/collection/collection.py b/openviking/storage/vectordb/collection/collection.pyindex a15804450..45954e8cb 100644--- a/openviking/storage/vectordb/collection/collection.py+++ b/openviking/storage/vectordb/collection/collection.py@@ -193,9 +193,9 @@ class Collection:Raises:AssertionError: If the provided `collection` is not an instance of ICollection."""- assert isinstance(- collection, ICollection- ), "collection must be an instance of CollectionInterface"+ assert isinstance(collection, ICollection), (+ "collection must be an instance of CollectionInterface"+ )self.__collection: Optional[ICollection] = collectiondef __del__(self):@@ -392,9 +392,7 @@ class Collection:"""if self.__collection is None:raise RuntimeError("Collection is closed")- return self.__collection.search_by_id(- index_name, id, limit, offset, filters, output_fields- )+ return self.__collection.search_by_id(index_name, id, limit, offset, filters, output_fields)def search_by_multimodal(self,@@ -459,9 +457,7 @@ class Collection:"""if self.__collection is None:raise RuntimeError("Collection is closed")- return self.__collection.search_by_random(- index_name, limit, offset, filters, output_fields- )+ return self.__collection.search_by_random(index_name, limit, offset, filters, output_fields)def search_by_scalar(self,diff --git a/openviking/storage/vectordb/collection/local_collection.py b/openviking/storage/vectordb/collection/local_collection.pyindex 99cb7cfdc..43604f838 100644--- a/openviking/storage/vectordb/collection/local_collection.py+++ b/openviking/storage/vectordb/collection/local_collection.py@@ -26,24 +26,21 @@ from openviking.storage.vectordb.meta.collection_meta import CollectionMeta, crefrom openviking.storage.vectordb.meta.index_meta import create_index_metafrom openviking.storage.vectordb.store.data import CandidateData, DeltaRecordfrom openviking.storage.vectordb.store.store import OpType-from openviking.storage.vectordb.store.store_manager import (- StoreManager,- create_store_manager,-)+from openviking.storage.vectordb.store.store_manager import StoreManager, create_store_managerfrom openviking.storage.vectordb.utils import validation+from openviking.storage.vectordb.utils.config_utils import get_config_valuefrom openviking.storage.vectordb.utils.constants import (DEFAULT_INDEX_MAINTENANCE_SECONDS,DEFAULT_TTL_CLEANUP_SECONDS,ENV_INDEX_MAINTENANCE_SECONDS,ENV_TTL_CLEANUP_SECONDS,+ STORAGE_DIR_NAME,AggregateKeys,SpecialFields,- STORAGE_DIR_NAME,)from openviking.storage.vectordb.utils.dict_utils import ThreadSafeDictManagerfrom openviking.storage.vectordb.utils.id_generator import generate_auto_idfrom openviking.storage.vectordb.utils.str_to_uint64 import str_to_uint64-from openviking.storage.vectordb.utils.config_utils import get_config_valuefrom openviking.storage.vectordb.vectorize.base import BaseVectorizerfrom openviking.storage.vectordb.vectorize.vectorizer import VectorizerAdapterfrom openviking.storage.vectordb.vectorize.vectorizer_factory import VectorizerFactory@@ -902,7 +899,9 @@ class PersistCollection(LocalCollection):for index_name in index_names:meta_path = os.path.join(self.index_dir, index_name, "index_meta.json")if not os.path.exists(meta_path):- logger.warning(f"Index metadata file not found at {meta_path}, skipping recovery for index {index_name}")+ logger.warning(+ f"Index metadata file not found at {meta_path}, skipping recovery for index {index_name}"+ )continuemeta = create_index_meta(self.meta, meta_path)# When recovering an existing index, pass initial_timestamp=0.diff --git a/openviking/storage/vectordb/collection/volcengine_collection.py b/openviking/storage/vectordb/collection/volcengine_collection.pyindex 7dac57dd5..7ddfa309e 100644--- a/openviking/storage/vectordb/collection/volcengine_collection.py+++ b/openviking/storage/vectordb/collection/volcengine_collection.py@@ -20,9 +20,7 @@ from openviking.storage.vectordb.collection.volcengine_clients import (from openviking.utils.logger import default_logger as logger-def get_or_create_volcengine_collection(- config: Dict[str, Any], meta_data: Dict[str, Any]-):+def get_or_create_volcengine_collection(config: Dict[str, Any], meta_data: Dict[str, Any]):"""Get or create a Volcengine Collection.diff --git a/openviking/storage/vectordb/index/local_index.py b/openviking/storage/vectordb/index/local_index.pyindex 0c642ff6f..4132b19ec 100644--- a/openviking/storage/vectordb/index/local_index.py+++ b/openviking/storage/vectordb/index/local_index.py@@ -11,7 +11,6 @@ from typing import Any, Dict, List, Optional, Tuple, Unionimport openviking.storage.vectordb.engine as enginefrom openviking.storage.vectordb.index.index import IIndexfrom openviking.storage.vectordb.store.data import CandidateData, DeltaRecord-from openviking.storage.vectordb.utils import file_utilsfrom openviking.storage.vectordb.utils.constants import IndexFileMarkersfrom openviking.utils.logger import default_logger as logger@@ -85,15 +84,15 @@ class IndexEngineProxy:query_vector = normalize_vector(query_vector)req.query = query_vectorreq.topk = limit-+if filters is None:filters = {}req.dsl = json.dumps(filters)-+if sparse_raw_terms and sparse_values:req.sparse_raw_terms = sparse_raw_termsreq.sparse_values = sparse_values-+search_result = self.index_engine.search(req)labels = search_result.labelsscores = search_result.scores@@ -102,7 +101,7 @@ class IndexEngineProxy:def add_data(self, cands_list: List[CandidateData]):if not self.index_engine:raise RuntimeError("Index engine not initialized")-+add_req_list = [engine.AddDataRequest() for _ in range(len(cands_list))]for i, data in enumerate(cands_list):add_req_list[i].label = data.label@@ -255,7 +254,7 @@ class LocalIndex(IIndex):sparse_raw_terms = []if sparse_values is None:sparse_values = []-+return self.engine_proxy.search(query_vector, limit, filters, sparse_raw_terms, sparse_values)@@ -267,7 +266,7 @@ class LocalIndex(IIndex):) -> Dict[str, Any]:if not self.engine_proxy or not self.engine_proxy.index_engine:return {}-+extra_json = ""try:req = engine.SearchRequest()@@ -366,7 +365,7 @@ class VolatileIndex(LocalIndex):"""if cands_list is None:cands_list = []-+index_config_dict = meta.get_build_index_dict()version_int = int(time.time_ns())index_config_dict["VectorIndex"]["ElementCount"] = len(cands_list)@@ -438,13 +437,13 @@ class PersistentIndex(LocalIndex):"""def __init__(- self,- name: str,- meta: Any,- path: str,- cands_list: Optional[List[CandidateData]] = None,+ self,+ name: str,+ meta: Any,+ path: str,+ cands_list: Optional[List[CandidateData]] = None,force_rebuild: bool = False,- initial_timestamp: Optional[int] = None+ initial_timestamp: Optional[int] = None,):"""Initialize a persistent index with versioning support.@@ -473,12 +472,12 @@ class PersistentIndex(LocalIndex):"""if cands_list is None:cands_list = []-+self.index_dir = os.path.join(path, name)os.makedirs(self.index_dir, exist_ok=True)self.version_dir = os.path.join(self.index_dir, "versions")os.makedirs(self.version_dir, exist_ok=True)-+newest_version = self.get_newest_version()# At this point, there is no index, need to create a new one@@ -491,11 +490,17 @@ class PersistentIndex(LocalIndex):super().__init__(index_path, meta)# Remove scheduling logic, unified scheduling by collection layer- def _create_new_index(self, name: str, meta: Any, cands_list: List[CandidateData], initial_timestamp: Optional[int] = None):+ def _create_new_index(+ self,+ name: str,+ meta: Any,+ cands_list: List[CandidateData],+ initial_timestamp: Optional[int] = None,+ ):"""Create a new index from scratch."""# Get the vector normalization flag from metanormalize_vector_flag = meta.inner_meta.get("VectorIndex", {}).get("NormalizeVector", False)-+version_int = initial_timestamp if initial_timestamp is not None else int(time.time_ns())version_str = str(version_int)index_config_dict = meta.get_build_index_dict()@@ -503,11 +508,11 @@ class PersistentIndex(LocalIndex):index_config_dict["VectorIndex"]["MaxElementCount"] = len(cands_list)index_config_dict["UpdateTimeStamp"] = version_intindex_config_json = json.dumps(index_config_dict)-+builder = IndexEngineProxy(index_config_json, normalize_vector_flag)build_index_path = os.path.join(self.version_dir, version_str)builder.add_data(cands_list)-+dump_version_int = builder.dump(build_index_path)if dump_version_int > 0:dump_version_str = str(dump_version_int)@@ -645,21 +650,21 @@ class PersistentIndex(LocalIndex):# Must be a directoryif not os.path.isdir(version_path):continue-+# Must be an integer (timestamp)if not name.isdigit():continue-+# Must have corresponding .write_done filemarker_path = version_path + IndexFileMarkers.WRITE_DONE.valueif not os.path.exists(marker_path):continue-+valid_versions.append(int(name))-+if not valid_versions:return 0-+return max(valid_versions)def drop(self):diff --git a/openviking/storage/vectordb/meta/collection_meta.py b/openviking/storage/vectordb/meta/collection_meta.pyindex 0c217c759..59a5ceb51 100644--- a/openviking/storage/vectordb/meta/collection_meta.py+++ b/openviking/storage/vectordb/meta/collection_meta.py@@ -1,7 +1,7 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0import copy-from typing import Any, Dict, List, Optional+from typing import Any, Dict, Optionalfrom openviking.storage.vectordb.meta.dict import IDictfrom openviking.storage.vectordb.meta.local_dict import PersistentDict, VolatileDict@@ -102,11 +102,7 @@ class CollectionMeta:"",)inner_meta["SparseVectorKey"] = next(- (- item["FieldName"]- for item in fields- if item.get("FieldType") == "sparse_vector"- ),+ (item["FieldName"] for item in fields if item.get("FieldType") == "sparse_vector"),"",)return inner_meta@@ -124,9 +120,7 @@ class CollectionMeta:user_meta = copy.deepcopy(inner_meta)if not user_meta.get("HasPrimaryKey", False):fields = user_meta.get("Fields", [])- new_list = [- item for item in fields if "AUTO_ID" != item.get("FieldName", "")- ]+ new_list = [item for item in fields if "AUTO_ID" != item.get("FieldName", "")]user_meta["Fields"] = new_listuser_meta.pop("HasPrimaryKey", None)user_meta.pop("FieldsCount", None)diff --git a/openviking/storage/vectordb/meta/index_meta.py b/openviking/storage/vectordb/meta/index_meta.pyindex 1704c2cc2..3c2253e72 100644--- a/openviking/storage/vectordb/meta/index_meta.py+++ b/openviking/storage/vectordb/meta/index_meta.py@@ -29,9 +29,7 @@ def create_index_meta("""inner_meta = {}if user_meta:- if not validation.is_valid_index_meta_data(- user_meta, collection_meta.fields_dict- ):+ if not validation.is_valid_index_meta_data(user_meta, collection_meta.fields_dict):raise ValueError("invalid index_meta {} fields_dict {}".format(user_meta, collection_meta.fields_dict@@ -120,9 +118,7 @@ class IndexMeta:if "flat" in inner_meta["VectorIndex"]["IndexType"]:vector_index["IndexType"] = "flat"if "EnableSparse" in inner_meta["VectorIndex"]:- vector_index["EnableSparse"] = inner_meta[- "VectorIndex"- ]["EnableSparse"]+ vector_index["EnableSparse"] = inner_meta["VectorIndex"]["EnableSparse"]if "SearchWithSparseLogitAlpha" in inner_meta["VectorIndex"]:vector_index["SearchWithSparseLogitAlpha"] = inner_meta["VectorIndex"]["SearchWithSparseLogitAlpha"diff --git a/openviking/storage/vectordb/project/project.py b/openviking/storage/vectordb/project/project.pyindex e06949822..1ed5d13c0 100644--- a/openviking/storage/vectordb/project/project.py+++ b/openviking/storage/vectordb/project/project.py@@ -1,165 +1,166 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0-class Project:- """Wrapper class for managing project operations.+from abc import ABC, abstractmethod+from typing import Any, Dict- A Project serves as a container for managing multiple collections. It provides- a unified interface for creating, accessing, and managing collections within- a project namespace.++class IProject(ABC):+ """Interface defining the contract for project implementations.++ All project implementations must inherit from this interface and implement+ all abstract methods for managing collections."""- def __init__(self, project):- """Initialize the Project wrapper.+ def __init__(self, project_name: str = "default"):+ """Initialize the project interface.Args:- project (IProject): An instance implementing the IProject interface.-- Raises:- AssertionError: If project is not an instance of IProject.+ project_name (str): Name of the project. Defaults to 'default'."""- assert isinstance(project, IProject), "project must be IProject"- self.__project = project+ self.project_name = project_name+ @abstractmethoddef close(self):- """Close the project and release all associated resources."""- self.__project.close()+ """Close the project and release resources.- def has_collection(self, collection_name):- """Check if a collection exists in the project.+ Must be implemented by subclasses to properly clean up resources.+ """+ pass++ @abstractmethod+ def has_collection(self, collection_name: str) -> bool:+ """Check if a collection exists.Args:collection_name (str): Name of the collection to check.Returns:- bool: True if the collection exists, False otherwise.+ bool: True if collection exists, False otherwise."""- return self.__project.has_collection(collection_name)+ pass- def get_collection(self, collection_name):+ @abstractmethod+ def get_collection(self, collection_name: str) -> Any:"""Retrieve a collection by name.Args:collection_name (str): Name of the collection to retrieve.Returns:- Collection: The requested collection instance, or None if not found.+ Collection: The collection instance, or None if not found."""- return self.__project.get_collection(collection_name)+ pass- def get_collections(self):+ @abstractmethod+ def get_collections(self) -> Dict[str, Any]:"""Get all collections in the project.Returns:- Dict[str, Collection]: Dictionary mapping collection names to Collection instances.+ Dict[str, Collection]: Mapping of collection names to Collection instances."""- return self.__project.get_collections()+ pass- def create_collection(self, collection_name, collection_meta):- """Create a new collection in the project.+ @abstractmethod+ def create_collection(self, collection_name: str, collection_meta: Dict[str, Any]) -> Any:+ """Create a new collection.Args:- collection_name (str): Name for the new collection. Must be unique within the project.- collection_meta (Dict[str, Any]): Metadata configuration for the collection, including- fields definition, primary key, vector configuration, etc.+ collection_name (str): Unique name for the collection.+ collection_meta (Dict[str, Any]): Collection metadata and configuration.Returns:Collection: The newly created collection instance.-- Raises:- ValueError: If a collection with the same name already exists or if the metadata is invalid."""- return self.__project.create_collection(collection_name, collection_meta)+ pass- def drop_collection(self, collection_name):- """Delete a collection from the project.+ @abstractmethod+ def drop_collection(self, collection_name: str):+ """Delete a collection.Args:collection_name (str): Name of the collection to delete.-- Note:- This operation is irreversible and will permanently delete all data and indexes- associated with the collection."""- return self.__project.drop_collection(collection_name)-+ pass-from abc import ABC, abstractmethod-from typing import Any, Dict, Optional-class IProject(ABC):- """Interface defining the contract for project implementations.+class Project:+ """Wrapper class for managing project operations.- All project implementations must inherit from this interface and implement- all abstract methods for managing collections.+ A Project serves as a container for managing multiple collections. It provides+ a unified interface for creating, accessing, and managing collections within+ a project namespace."""- def __init__(self, project_name: str = "default"):- """Initialize the project interface.+ def __init__(self, project):+ """Initialize the Project wrapper.Args:- project_name (str): Name of the project. Defaults to 'default'.+ project (IProject): An instance implementing the IProject interface.++ Raises:+ AssertionError: If project is not an instance of IProject."""- self.project_name = project_name+ assert isinstance(project, IProject), "project must be IProject"+ self.__project = project- @abstractmethoddef close(self):- """Close the project and release resources.-- Must be implemented by subclasses to properly clean up resources.- """- pass+ """Close the project and release all associated resources."""+ self.__project.close()- @abstractmethod- def has_collection(self, collection_name: str) -> bool:- """Check if a collection exists.+ def has_collection(self, collection_name):+ """Check if a collection exists in the project.Args:collection_name (str): Name of the collection to check.Returns:- bool: True if collection exists, False otherwise.+ bool: True if the collection exists, False otherwise."""- pass+ return self.__project.has_collection(collection_name)- @abstractmethod- def get_collection(self, collection_name: str) -> Any:+ def get_collection(self, collection_name):"""Retrieve a collection by name.Args:collection_name (str): Name of the collection to retrieve.Returns:- Collection: The collection instance, or None if not found.+ Collection: The requested collection instance, or None if not found."""- pass+ return self.__project.get_collection(collection_name)- @abstractmethod- def get_collections(self) -> Dict[str, Any]:+ def get_collections(self):"""Get all collections in the project.Returns:- Dict[str, Collection]: Mapping of collection names to Collection instances.+ Dict[str, Collection]: Dictionary mapping collection names to Collection instances."""- pass+ return self.__project.get_collections()- @abstractmethod- def create_collection(self, collection_name: str, collection_meta: Dict[str, Any]) -> Any:- """Create a new collection.+ def create_collection(self, collection_name, collection_meta):+ """Create a new collection in the project.Args:- collection_name (str): Unique name for the collection.- collection_meta (Dict[str, Any]): Collection metadata and configuration.+ collection_name (str): Name for the new collection. Must be unique within the project.+ collection_meta (Dict[str, Any]): Metadata configuration for the collection, including+ fields definition, primary key, vector configuration, etc.Returns:Collection: The newly created collection instance.++ Raises:+ ValueError: If a collection with the same name already exists or if the metadata is invalid."""- pass+ return self.__project.create_collection(collection_name, collection_meta)- @abstractmethod- def drop_collection(self, collection_name: str):- """Delete a collection.+ def drop_collection(self, collection_name):+ """Delete a collection from the project.Args:collection_name (str): Name of the collection to delete.++ Note:+ This operation is irreversible and will permanently delete all data and indexes+ associated with the collection."""- pass+ return self.__project.drop_collection(collection_name)diff --git a/openviking/storage/vectordb/service/api_fastapi.py b/openviking/storage/vectordb/service/api_fastapi.pyindex aae163c5e..52230b93e 100644--- a/openviking/storage/vectordb/service/api_fastapi.py+++ b/openviking/storage/vectordb/service/api_fastapi.py@@ -5,8 +5,7 @@ import timefrom dataclasses import asdictfrom typing import Any, Optional-from fastapi import APIRouter, Query, Request, Depends, HTTPException-from fastapi.responses import JSONResponse+from fastapi import APIRouter, Depends, Query, Requestfrom openviking.storage.vectordb.project.project_group import get_or_create_project_groupfrom openviking.storage.vectordb.service.app_models import (@@ -28,7 +27,6 @@ from openviking.storage.vectordb.service.app_models import ()from openviking.storage.vectordb.service.code import ErrorCodefrom openviking.storage.vectordb.utils import data_utils-from openviking.storage.vectordb.utils.api_utils import form_error, form_resfrom openviking.utils.logger import default_logger as logger@@ -85,34 +83,38 @@ search_router = APIRouter(prefix="/api/vikingdb/data/search", tags=["Search"])# ==================== Dependencies ====================+def get_project(project_name: str = "default"):"""Get project instance"""return project_group.get_or_create_project(project_name)+def get_collection_or_raise(collection_name: str, project_name: str = "default"):"""Get collection instance or raise exception if not found"""if not collection_name:raise VikingDBException(ErrorCode.INVALID_PARAM, "collection name is empty")-+project = project_group.get_or_create_project(project_name)collection = project.get_collection(collection_name)if not collection:raise VikingDBException(ErrorCode.COLLECTION_NOT_EXIST, "collection not exist")return collection+# Dependency for GET requests using Query parametersdef get_collection_dependency(CollectionName: str = Query(..., description="Collection name"),- ProjectName: Optional[str] = Query("default", description="Project name")+ ProjectName: Optional[str] = Query("default", description="Project name"),):if not CollectionName:raise VikingDBException(ErrorCode.INVALID_PARAM, "collection name is empty")return get_collection_or_raise(CollectionName, ProjectName)+# Dependency for snake_case query paramsdef get_collection_dependency_snake(collection_name: str = Query(..., description="Collection name"),- project: Optional[str] = Query("default", description="Project name")+ project: Optional[str] = Query("default", description="Project name"),):if not collection_name:raise VikingDBException(ErrorCode.INVALID_PARAM, "collection name is empty")@@ -164,8 +166,12 @@ async def update_collection(request: CollectionUpdateRequest, req: Request):"""Update an existing collection"""try:if not request.CollectionName:- return error_response("CollectionName is empty", ErrorCode.INVALID_PARAM.value, request=req)- collection = get_collection_or_raise(request.CollectionName, request.ProjectName or "default")+ return error_response(+ "CollectionName is empty", ErrorCode.INVALID_PARAM.value, request=req+ )+ collection = get_collection_or_raise(+ request.CollectionName, request.ProjectName or "default"+ )description = request.Descriptionfields = data_utils.convert_dict(request.Fields)collection.update(fields, description)@@ -175,10 +181,7 @@ async def update_collection(request: CollectionUpdateRequest, req: Request):@collection_router.get("/GetVikingdbCollection", response_model=ApiResponse)-async def get_collection_info(- req: Request,- collection: Any = Depends(get_collection_dependency)-):+async def get_collection_info(req: Request, collection: Any = Depends(get_collection_dependency)):"""Get collection information"""meta_data = collection.get_meta_data()return success_response("collection info", meta_data, request=req)@@ -229,7 +232,7 @@ async def upsert_data(request: DataUpsertRequest, req: Request):"""Upsert data to collection"""try:collection = get_collection_or_raise(request.collection_name, request.project or "default")-+ttl = request.ttl or 0data_list = data_utils.convert_dict(request.fields)@@ -247,7 +250,7 @@ async def upsert_data(request: DataUpsertRequest, req: Request):async def fetch_data(req: Request,ids: str = Query(..., description="Primary key list"),- collection: Any = Depends(get_collection_dependency_snake)+ collection: Any = Depends(get_collection_dependency_snake),):"""Fetch data from collection"""primary_keys = data_utils.convert_dict(ids)@@ -260,7 +263,7 @@ async def delete_data(request: DataDeleteRequest, req: Request):"""Delete data from collection"""try:collection = get_collection_or_raise(request.collection_name, request.project or "default")-+primary_keys = data_utils.convert_dict(request.ids) if request.ids else []del_all = request.del_all or False@@ -281,8 +284,10 @@ async def delete_data(request: DataDeleteRequest, req: Request):async def create_index(request: IndexCreateRequest, req: Request):"""Create an index"""try:- collection = get_collection_or_raise(request.CollectionName, request.ProjectName or "default")-+ collection = get_collection_or_raise(+ request.CollectionName, request.ProjectName or "default"+ )+index_name = request.IndexNameif not index_name:return error_response("index name is empty", ErrorCode.INVALID_PARAM.value, request=req)@@ -312,8 +317,10 @@ async def create_index(request: IndexCreateRequest, req: Request):async def update_index(request: IndexUpdateRequest, req: Request):"""Update an index"""try:- collection = get_collection_or_raise(request.CollectionName, request.ProjectName or "default")-+ collection = get_collection_or_raise(+ request.CollectionName, request.ProjectName or "default"+ )+index_name = request.IndexNameif not index_name:return error_response("index name is empty", ErrorCode.INVALID_PARAM.value, request=req)@@ -331,7 +338,7 @@ async def update_index(request: IndexUpdateRequest, req: Request):async def get_index_info(req: Request,IndexName: str = Query(..., description="Index name"),- collection: Any = Depends(get_collection_dependency)+ collection: Any = Depends(get_collection_dependency),):"""Get index information"""if not IndexName:@@ -342,10 +349,7 @@ async def get_index_info(@index_router.get("/ListVikingdbIndex", response_model=ApiResponse)-async def list_indexes(- req: Request,- collection: Any = Depends(get_collection_dependency)-):+async def list_indexes(req: Request, collection: Any = Depends(get_collection_dependency)):"""List all indexes"""data = collection.list_indexes()return success_response("list indexes success", data, request=req)@@ -355,8 +359,10 @@ async def list_indexes(async def drop_index(request: IndexDropRequest, req: Request):"""Delete an index"""try:- collection = get_collection_or_raise(request.CollectionName, request.ProjectName or "default")-+ collection = get_collection_or_raise(+ request.CollectionName, request.ProjectName or "default"+ )+index_name = request.IndexNameif not index_name:return error_response("index name is empty", ErrorCode.INVALID_PARAM.value, request=req)@@ -375,7 +381,7 @@ async def search_by_vector(request: SearchByVectorRequest, req: Request):"""Search by vector"""try:collection = get_collection_or_raise(request.collection_name, request.project or "default")-+index_name = request.index_nameif not index_name:return error_response("index name is empty", ErrorCode.INVALID_PARAM.value, request=req)@@ -392,7 +398,9 @@ async def search_by_vector(request: SearchByVectorRequest, req: Request):offset = request.offset or 0if offset < 0:return error_response(- "offset must be greater than or equal to 0", ErrorCode.INVALID_PARAM.value, request=req+ "offset must be greater than or equal to 0",+ ErrorCode.INVALID_PARAM.value,+ request=req,)result = collection.search_by_vector(@@ -414,7 +422,7 @@ async def search_by_id(request: SearchByIdRequest, req: Request):"""Search by ID"""try:collection = get_collection_or_raise(request.collection_name, request.project or "default")-+index_name = request.index_nameif not index_name:return error_response("index name is empty", ErrorCode.INVALID_PARAM.value, request=req)@@ -433,7 +441,9 @@ async def search_by_id(request: SearchByIdRequest, req: Request):offset = request.offset or 0if offset < 0:return error_response(- "offset must be greater than or equal to 0", ErrorCode.INVALID_PARAM.value, request=req+ "offset must be greater than or equal to 0",+ ErrorCode.INVALID_PARAM.value,+ request=req,)result = collection.search_by_id(@@ -454,7 +464,7 @@ async def search_by_multimodal(request: SearchByMultiModalRequest, req: Request)"""Search by multimodal"""try:collection = get_collection_or_raise(request.collection_name, request.project or "default")-+index_name = request.index_nameif not index_name:return error_response("index name is empty", ErrorCode.INVALID_PARAM.value, request=req)@@ -480,7 +490,9 @@ async def search_by_multimodal(request: SearchByMultiModalRequest, req: Request)offset = request.offset or 0if offset < 0:return error_response(- "offset must be greater than or equal to 0", ErrorCode.INVALID_PARAM.value, request=req+ "offset must be greater than or equal to 0",+ ErrorCode.INVALID_PARAM.value,+ request=req,)try:@@ -507,7 +519,7 @@ async def search_by_scalar(request: SearchByScalarRequest, req: Request):"""Search by scalar field"""try:collection = get_collection_or_raise(request.collection_name, request.project or "default")-+index_name = request.index_nameif not index_name:return error_response("index name is empty", ErrorCode.INVALID_PARAM.value, request=req)@@ -527,7 +539,9 @@ async def search_by_scalar(request: SearchByScalarRequest, req: Request):offset = request.offset or 0if offset < 0:return error_response(- "offset must be greater than or equal to 0", ErrorCode.INVALID_PARAM.value, request=req+ "offset must be greater than or equal to 0",+ ErrorCode.INVALID_PARAM.value,+ request=req,)result = collection.search_by_scalar(@@ -549,7 +563,7 @@ async def search_by_random(request: SearchByRandomRequest, req: Request):"""Search by random"""try:collection = get_collection_or_raise(request.collection_name, request.project or "default")-+index_name = request.index_nameif not index_name:return error_response("index name is empty", ErrorCode.INVALID_PARAM.value, request=req)@@ -564,7 +578,9 @@ async def search_by_random(request: SearchByRandomRequest, req: Request):offset = request.offset or 0if offset < 0:return error_response(- "offset must be greater than or equal to 0", ErrorCode.INVALID_PARAM.value, request=req+ "offset must be greater than or equal to 0",+ ErrorCode.INVALID_PARAM.value,+ request=req,)result = collection.search_by_random(@@ -584,7 +600,7 @@ async def search_by_keywords(request: SearchByKeywordsRequest, req: Request):"""Search by keywords"""try:collection = get_collection_or_raise(request.collection_name, request.project or "default")-+index_name = request.index_nameif not index_name:return error_response("index name is empty", ErrorCode.INVALID_PARAM.value, request=req)@@ -609,7 +625,9 @@ async def search_by_keywords(request: SearchByKeywordsRequest, req: Request):offset = request.offset or 0if offset < 0:return error_response(- "offset must be greater than or equal to 0", ErrorCode.INVALID_PARAM.value, request=req+ "offset must be greater than or equal to 0",+ ErrorCode.INVALID_PARAM.value,+ request=req,)try:diff --git a/openviking/storage/vectordb/service/server_fastapi.py b/openviking/storage/vectordb/service/server_fastapi.pyindex fa010537d..d63914974 100644--- a/openviking/storage/vectordb/service/server_fastapi.py+++ b/openviking/storage/vectordb/service/server_fastapi.py@@ -45,8 +45,7 @@ app = FastAPI(@app.exception_handler(VikingDBException)async def vikingdb_exception_handler(request: Request, exc: VikingDBException):return JSONResponse(- status_code=200,- content=error_response(exc.message, exc.code.value, request=request)+ status_code=200, content=error_response(exc.message, exc.code.value, request=request))diff --git a/openviking/storage/vectordb/store/bytes_row.py b/openviking/storage/vectordb/store/bytes_row.pyindex 043a638a3..61eb178df 100644--- a/openviking/storage/vectordb/store/bytes_row.py+++ b/openviking/storage/vectordb/store/bytes_row.py@@ -211,7 +211,7 @@ class BytesRow:field_meta = self.schema.get_field_meta(field_name)if field_meta.id >= serialized_data[0]:return field_meta.default_value-+# Use '<' for little-endian in all unpack operationsif field_meta.data_type == FieldType.int64:return struct.unpack_from("<q", serialized_data, field_meta.offset)[0]diff --git a/openviking/storage/vectordb/store/local_store.py b/openviking/storage/vectordb/store/local_store.pyindex 106a0dc8c..b1ab54cb5 100644--- a/openviking/storage/vectordb/store/local_store.py+++ b/openviking/storage/vectordb/store/local_store.py@@ -1,13 +1,13 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0-import time-from typing import List, Tuple, Union, Any+from typing import List, Tuple, Unionimport openviking.storage.vectordb.engine as enginefrom openviking.storage.vectordb.store.store import BatchOp, IMutiTableStore, Op, OpType# Constant for the maximum Unicode character, used for range queries to cover all possible keys-MAX_UNICODE_CHAR = "\U0010FFFF"+MAX_UNICODE_CHAR = "\U0010ffff"+def create_store_engine_proxy(path: str = "") -> "StoreEngineProxy":"""Create a storage engine proxy.@@ -175,9 +175,7 @@ class StoreEngineProxy(IMutiTableStore):# batch_op.op_type can be a list or a single valueif isinstance(batch_op.op_type, list):op_type = (- batch_op.op_type[i]- if i < len(batch_op.op_type)- else batch_op.op_type[0]+ batch_op.op_type[i] if i < len(batch_op.op_type) else batch_op.op_type[0])else:op_type = batch_op.op_type@@ -189,8 +187,6 @@ class StoreEngineProxy(IMutiTableStore):engine_op.key = batch_op.table + key# Safety check for data_list- engine_op.value = (- batch_op.data_list[i] if i < len(batch_op.data_list) else ""- )+ engine_op.value = batch_op.data_list[i] if i < len(batch_op.data_list) else ""engine_op_list.append(engine_op)self.storage_engine.exec_op(engine_op_list)diff --git a/openviking/storage/vectordb/store/serializable.py b/openviking/storage/vectordb/store/serializable.pyindex 185464c20..f2939a63d 100644--- a/openviking/storage/vectordb/store/serializable.py+++ b/openviking/storage/vectordb/store/serializable.py@@ -3,9 +3,11 @@"""Elegant serialization decorator: automatically generate schema and serialization methods from type annotations"""-from dataclasses import fields, is_dataclass, asdict-from typing import Any, get_origin, get_args-from openviking.storage.vectordb.store.bytes_row import Schema, BytesRow, FieldType++from dataclasses import asdict, fields, is_dataclass+from typing import Any, get_args, get_origin++from openviking.storage.vectordb.store.bytes_row import BytesRow, FieldType, Schemadef _python_type_to_field_type(py_type: Any, field_name: str) -> FieldType:diff --git a/openviking/storage/vectordb/store/store_manager.py b/openviking/storage/vectordb/store/store_manager.pyindex 8a9db605a..36ba6bed1 100644--- a/openviking/storage/vectordb/store/store_manager.py+++ b/openviking/storage/vectordb/store/store_manager.py@@ -1,7 +1,7 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0import time-from typing import List, Optional, Tuple, Union+from typing import List, Optional, Tuplefrom openviking.storage.vectordb.store.data import CandidateData, DeltaRecordfrom openviking.storage.vectordb.store.local_store import create_store_engine_proxy@@ -80,9 +80,7 @@ class StoreManager:)for bytes_data in bytes_list]- delta_list = [- DeltaRecord(type=DeltaRecord.Type.UPSERT) for _ in range(len(cands_list))- ]+ delta_list = [DeltaRecord(type=DeltaRecord.Type.UPSERT) for _ in range(len(cands_list))]for i, old_fields in enumerate(old_cands_fields_list):delta_list[i].label = cands_list[i].labeldelta_list[i].vector = cands_list[i].vector@@ -154,9 +152,7 @@ class StoreManager:)for bytes_data in bytes_list]- delta_list = [- DeltaRecord(type=DeltaRecord.Type.DELETE) for _ in range(len(label_list))- ]+ delta_list = [DeltaRecord(type=DeltaRecord.Type.DELETE) for _ in range(len(label_list))]for i, data in enumerate(old_cands_fields_list):delta_list[i].label = label_list[i]delta_list[i].old_fields = data@@ -181,9 +177,7 @@ class StoreManager:self.storage.exec_sequence_batch_op(batch_op_list)return delta_list- def fetch_cands_data(- self, label_list: List[int]- ) -> List[Optional[CandidateData]]:+ def fetch_cands_data(self, label_list: List[int]) -> List[Optional[CandidateData]]:"""Fetch candidate data by labels.Args:@@ -272,13 +266,13 @@ class StoreManager:# Optimize: Avoid full deserialization if only checking expire_ns_ts# But we need label and fields later for DeltaRecord.# Let's filter first by expire_ns_ts which is in CandidateData.-+expired_cands_data: List[Tuple[int, str]] = []-+for byte_data in cands_bytes_list:if not byte_data:continue-+# Efficiently check expiration without full object creation if possible# But CandidateData takes bytes_data in init, so we might as well use helperexpire_ts = CandidateData.bytes_row.deserialize_field(byte_data, "expire_ns_ts")@@ -291,7 +285,7 @@ class StoreManager:delta_list = [DeltaRecord(type=DeltaRecord.Type.DELETE) for _ in range(len(expired_cands_data))]-+if expired_cands_data:batch_op_list.append(BatchOp(@@ -304,7 +298,7 @@ class StoreManager:for i, data in enumerate(expired_cands_data):delta_list[i].label = data[0]delta_list[i].old_fields = data[1]-+base_ts = time.time_ns()batch_op_list.append(BatchOp(@@ -324,6 +318,6 @@ class StoreManager:["" for _ in range(len(ttl_kv_list))],))-+self.storage.exec_sequence_batch_op(batch_op_list)return delta_listdiff --git a/openviking/storage/vectordb/utils/api_utils.py b/openviking/storage/vectordb/utils/api_utils.pyindex c5924d337..58b31c052 100644--- a/openviking/storage/vectordb/utils/api_utils.py+++ b/openviking/storage/vectordb/utils/api_utils.py@@ -2,6 +2,7 @@# SPDX-License-Identifier: Apache-2.0from typing import Any, Dict, Optional+def form_error(err: Any, code: int = 400, data: Optional[Dict] = None) -> tuple:if data is None:data = {}diff --git a/openviking/storage/vectordb/utils/config_utils.py b/openviking/storage/vectordb/utils/config_utils.pyindex 7457032fd..785ca8df4 100644--- a/openviking/storage/vectordb/utils/config_utils.py+++ b/openviking/storage/vectordb/utils/config_utils.py@@ -3,6 +3,7 @@import osfrom typing import Any, Dict, Optional+def get_config_value(config: Optional[Dict[str, Any]], config_key: str, env_var: str, default_value: Any) -> Any:diff --git a/openviking/storage/vectordb/utils/data_utils.py b/openviking/storage/vectordb/utils/data_utils.pyindex 06a039526..cb8db6fe3 100644--- a/openviking/storage/vectordb/utils/data_utils.py+++ b/openviking/storage/vectordb/utils/data_utils.py@@ -2,7 +2,8 @@# SPDX-License-Identifier: Apache-2.0import jsonimport logging-from typing import Any, Dict, List, Union, Optional+from typing import Dict, List, Union+def convert_dict(p: Union[str, Dict, List], no_exception: bool = False) -> Union[Dict, List]:"""diff --git a/openviking/storage/vectordb/utils/dict_utils.py b/openviking/storage/vectordb/utils/dict_utils.pyindex fbf352d8a..c76b14a9d 100644--- a/openviking/storage/vectordb/utils/dict_utils.py+++ b/openviking/storage/vectordb/utils/dict_utils.py@@ -132,8 +132,7 @@ def filter_dict_key_with_prefix(d: Dict[str, Any], prefix: str = "_") -> Dict[stfiltered[key] = filter_dict_key_with_prefix(value, prefix)elif isinstance(value, list):filtered[key] = [- filter_dict_key_with_prefix(v, prefix) if isinstance(v, dict) else v- for v in value+ filter_dict_key_with_prefix(v, prefix) if isinstance(v, dict) else v for v in value]else:filtered[key] = valuediff --git a/openviking/storage/vectordb/utils/file_utils.py b/openviking/storage/vectordb/utils/file_utils.pyindex 206c4458d..95487fd98 100644--- a/openviking/storage/vectordb/utils/file_utils.py+++ b/openviking/storage/vectordb/utils/file_utils.py@@ -2,6 +2,7 @@# SPDX-License-Identifier: Apache-2.0import logging+def touch_file(file_path: str) -> None:try:with open(file_path, "a"):diff --git a/openviking/storage/vectordb/utils/id_generator.py b/openviking/storage/vectordb/utils/id_generator.pyindex 52f4f4eb7..39ae5b581 100644--- a/openviking/storage/vectordb/utils/id_generator.py+++ b/openviking/storage/vectordb/utils/id_generator.py@@ -1,37 +1,38 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0-import time-import threadingimport osimport random+import threading+import time+class SnowflakeGenerator:"""Distributed unique ID generator based on Twitter's Snowflake algorithm.Generates 64-bit integers (int64/uint64 compatible).-+Structure (64 bits):- 1 bit: Unused (sign bit)- 41 bits: Timestamp (milliseconds since epoch)- 10 bits: Machine/Process ID (5 bits datacenter + 5 bits worker)- 12 bits: Sequence number (per millisecond)"""-+# ConstantsEPOCH = 1704067200000 # 2024-01-01 00:00:00 UTC-+worker_id_bits = 5datacenter_id_bits = 5sequence_bits = 12-+max_worker_id = -1 ^ (-1 << worker_id_bits)max_datacenter_id = -1 ^ (-1 << datacenter_id_bits)max_sequence = -1 ^ (-1 << sequence_bits)-+worker_id_shift = sequence_bitsdatacenter_id_shift = sequence_bits + worker_id_bitstimestamp_left_shift = sequence_bits + worker_id_bits + datacenter_id_bits-+def __init__(self, worker_id: int = None, datacenter_id: int = None):"""Initialize the generator.@@ -41,46 +42,48 @@ class SnowflakeGenerator:# Use Process ID to distinguish processes on the same machine# PID can be large, so we mask it to fit in worker_id_bitsworker_id = os.getpid() & self.max_worker_id-+if datacenter_id is None:# In a containerized environment, hostname usually changes.# Using hash of hostname or a random number if not configured.# For local single-node usage, random is acceptable initialization.datacenter_id = random.randint(0, self.max_datacenter_id)-+if worker_id > self.max_worker_id or worker_id < 0:raise ValueError(f"worker_id must be between 0 and {self.max_worker_id}")if datacenter_id > self.max_datacenter_id or datacenter_id < 0:raise ValueError(f"datacenter_id must be between 0 and {self.max_datacenter_id}")-+self.worker_id = worker_idself.datacenter_id = datacenter_id-+self.sequence = 0self.last_timestamp = -1self.lock = threading.Lock()-+def _current_timestamp(self):return int(time.time() * 1000)-+def next_id(self) -> int:"""Generate the next unique ID."""with self.lock:timestamp = self._current_timestamp()-+if timestamp < self.last_timestamp:# Clock moved backwards, refuse to generate id# Wait until clock catches up or throw erroroffset = self.last_timestamp - timestamp- if offset <= 5: # If offset is small, just wait+ if offset <= 5: # If offset is small, just waittime.sleep(offset / 1000.0 + 0.001)timestamp = self._current_timestamp()-+if timestamp < self.last_timestamp:- raise Exception(f"Clock moved backwards. Refusing to generate id for {self.last_timestamp - timestamp} milliseconds")-+ raise Exception(+ f"Clock moved backwards. Refusing to generate id for {self.last_timestamp - timestamp} milliseconds"+ )+if self.last_timestamp == timestamp:self.sequence = (self.sequence + 1) & self.max_sequenceif self.sequence == 0:@@ -89,19 +92,23 @@ class SnowflakeGenerator:timestamp = self._current_timestamp()else:self.sequence = 0-+self.last_timestamp = timestamp-- new_id = ((timestamp - self.EPOCH) << self.timestamp_left_shift) | \- (self.datacenter_id << self.datacenter_id_shift) | \- (self.worker_id << self.worker_id_shift) | \- self.sequence-++ new_id = (+ ((timestamp - self.EPOCH) << self.timestamp_left_shift)+ | (self.datacenter_id << self.datacenter_id_shift)+ | (self.worker_id << self.worker_id_shift)+ | self.sequence+ )+return new_id+# Global instance_default_generator = SnowflakeGenerator()+def generate_auto_id() -> int:"""Generate a globally unique 64-bit integer ID.diff --git a/openviking/storage/vectordb/utils/logging_init.py b/openviking/storage/vectordb/utils/logging_init.pyindex 049f062a4..75333e595 100644--- a/openviking/storage/vectordb/utils/logging_init.py+++ b/openviking/storage/vectordb/utils/logging_init.py@@ -5,7 +5,6 @@import sysimport threading-_cpp_logging_initialized = False_cpp_logging_lock = threading.Lock()diff --git a/openviking/storage/vectordb/utils/validation.py b/openviking/storage/vectordb/utils/validation.pyindex 68bdd11c5..b42029824 100644--- a/openviking/storage/vectordb/utils/validation.py+++ b/openviking/storage/vectordb/utils/validation.py@@ -1,29 +1,24 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0from enum import Enum-from typing import Any, Dict, List, Literal, Optional, Union, Annotated-import random--from pydantic import (- BaseModel,- Field,- ValidationError as PydanticValidationError,- field_validator,- model_validator,- StringConstraints,- ConfigDict-)-from openviking.utils.logger import default_logger as logger+from typing import Annotated, Any, List, Literal, Optional++from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator+from pydantic import ValidationError as PydanticValidationError+from openviking.storage.vectordb.utils.id_generator import generate_auto_id+# Custom ValidationError for compatibilityclass ValidationError(Exception):def __init__(self, message: str, field_path: str = None):self.field_path = field_pathsuper().__init__(message)+# --- Basic Validators ---+def validate_name_str(name: str) -> str:if not name:raise ValueError("name is empty or None")@@ -33,13 +28,21 @@ def validate_name_str(name: str) -> str:raise ValueError(f"name must start with a letter, got '{name[0]}'")invalid_chars = [c for c in name if not (c.isalnum() or c == "_")]if invalid_chars:- raise ValueError(f"name can only contain letters, numbers and underscore, found invalid characters: {invalid_chars}")+ raise ValueError(+ f"name can only contain letters, numbers and underscore, found invalid characters: {invalid_chars}"+ )return name-ValidName = Annotated[str, Field(min_length=1, max_length=128), field_validator('name', mode='before', check_fields=False)(validate_name_str)]++ValidName = Annotated[+ str,+ Field(min_length=1, max_length=128),+ field_validator("name", mode="before", check_fields=False)(validate_name_str),+]# --- Models ---+class FieldTypeEnum(str, Enum):INT64 = "int64"FLOAT32 = "float32"@@ -56,9 +59,10 @@ class FieldTypeEnum(str, Enum):DATE_TIME = "date_time"GEO_POINT = "geo_point"+class DenseVectorize(BaseModel):- model_config = ConfigDict(extra='forbid')-+ model_config = ConfigDict(extra="forbid")+ModelName: strModelVersion: Optional[str] = NoneTextField: Optional[str] = None@@ -67,66 +71,69 @@ class DenseVectorize(BaseModel):Dim: Optional[int] = NoneDimension: Optional[int] = None- @field_validator('TextField', 'ImageField', 'VideoField')+ @field_validator("TextField", "ImageField", "VideoField")@classmethoddef check_fields(cls, v):- # We enforce presence logic in model_validator if needed,+ # We enforce presence logic in model_validator if needed,# but the original code had strict checks inside validate_dense_vectorize# Original: "TextField" is requiredreturn v-- @model_validator(mode='after')++ @model_validator(mode="after")def check_required(self):if self.TextField is None and self.ImageField is None and self.VideoField is None:- # Original logic: if "text_field" not in vectorize["dense"]- # The old code strictly required "TextField" (or "text_field" in logic, but schema used "TextField")- # Actually, old code: if "text_field" not in vectorize["dense"]: return False- # But ALLOWED keys used "TextField". It seems there's a case sensitivity issue in old code or intended.- # The old code check keys: 'ModelName', 'TextField' against ALLOWED_COLLECTION_DENSE_VECTORIZE_CHECK- # Let's assume PascalCase as per ALLOWED dictionary keys.- if not self.TextField:- raise ValueError("vectorize dense must contain TextField")+ # Original logic: if "text_field" not in vectorize["dense"]+ # The old code strictly required "TextField" (or "text_field" in logic, but schema used "TextField")+ # Actually, old code: if "text_field" not in vectorize["dense"]: return False+ # But ALLOWED keys used "TextField". It seems there's a case sensitivity issue in old code or intended.+ # The old code check keys: 'ModelName', 'TextField' against ALLOWED_COLLECTION_DENSE_VECTORIZE_CHECK+ # Let's assume PascalCase as per ALLOWED dictionary keys.+ if not self.TextField:+ raise ValueError("vectorize dense must contain TextField")return self+class SparseVectorize(BaseModel):- model_config = ConfigDict(extra='forbid')-+ model_config = ConfigDict(extra="forbid")+ModelName: strModelVersion: Optional[str] = NoneTextField: Optional[str] = None+class VectorizeConfig(BaseModel):- model_config = ConfigDict(extra='forbid')-+ model_config = ConfigDict(extra="forbid")+Dense: Optional[DenseVectorize] = NoneSparse: Optional[SparseVectorize] = None+class CollectionField(BaseModel):- model_config = ConfigDict(extra='forbid')-+ model_config = ConfigDict(extra="forbid")+FieldName: strFieldType: FieldTypeEnumDim: Optional[int] = Field(None, ge=4, le=4096)IsPrimaryKey: Optional[bool] = FalseDefaultValue: Optional[Any] = None-+# Internal fields_FieldID: Optional[int] = None- @field_validator('FieldName')+ @field_validator("FieldName")@classmethoddef validate_fieldname(cls, v):return validate_name_str(v)- @field_validator('Dim')+ @field_validator("Dim")@classmethoddef validate_dim(cls, v):if v is not None:if v % 4 != 0:raise ValueError(f"dimension must be a multiple of 4, got {v}")return v-- @model_validator(mode='after')++ @model_validator(mode="after")def validate_field_logic(cls, m):if m.FieldType == FieldTypeEnum.VECTOR:if m.Dim is None:@@ -136,25 +143,27 @@ class CollectionField(BaseModel):raise ValueError(f"primary key must be int64 or string, got '{m.FieldType}'")return m+class CollectionMetaConfig(BaseModel):- model_config = ConfigDict(extra='forbid')-+ model_config = ConfigDict(extra="forbid")+CollectionName: strFields: List[CollectionField]ProjectName: Optional[str] = NoneDescription: Optional[str] = Field(None, max_length=65535)Vectorize: Optional[VectorizeConfig] = None-+# Internal fields_FieldsCount: Optional[int] = None- @field_validator('CollectionName', 'ProjectName')+ @field_validator("CollectionName", "ProjectName")@classmethoddef validate_names(cls, v):- if v is None: return v+ if v is None:+ return vreturn validate_name_str(v)- @field_validator('Fields')+ @field_validator("Fields")@classmethoddef validate_fields_list(cls, fields):names = set()@@ -169,9 +178,10 @@ class CollectionMetaConfig(BaseModel):has_pk = Truereturn fields+class VectorIndexConfig(BaseModel):- model_config = ConfigDict(extra='forbid')-+ model_config = ConfigDict(extra="forbid")+IndexType: Literal["flat", "flat_hybrid", "FLAT", "FLAT_HYBRID"]Distance: Optional[Literal["l2", "ip", "cosine", "L2", "IP", "COSINE"]] = NoneQuant: Optional[Literal["int8", "float", "fix16", "pq", "INT8", "FLOAT", "FIX16", "PQ"]] = None@@ -183,7 +193,7 @@ class VectorIndexConfig(BaseModel):IndexWithSparseLogitAlpha: Optional[float] = NoneEnableSparse: Optional[bool] = None- @field_validator('IndexType', 'Distance', 'Quant', mode='before')+ @field_validator("IndexType", "Distance", "Quant", mode="before")@classmethoddef case_insensitive(cls, v):if isinstance(v, str):@@ -191,33 +201,34 @@ class VectorIndexConfig(BaseModel):# Actually user input might be mixed. Pydantic validates against Literals.# Let's normalize everything to lower case if the Literal allows it, or just pass through.# The old code did .lower() checks.- pass+ passreturn v-- @field_validator('IndexType')++ @field_validator("IndexType")@classmethoddef validate_index_type(cls, v):if v.lower() not in ["flat", "flat_hybrid"]:- raise ValueError(f"invalid index type '{v}'")+ raise ValueError(f"invalid index type '{v}'")return v- @field_validator('Distance')+ @field_validator("Distance")@classmethoddef validate_distance(cls, v):if v and v.lower() not in ["l2", "ip", "cosine"]:- raise ValueError(f"invalid distance type '{v}'")+ raise ValueError(f"invalid distance type '{v}'")return v-- @field_validator('Quant')++ @field_validator("Quant")@classmethoddef validate_quant(cls, v):if v and v.lower() not in ["int8", "float", "fix16", "pq"]:- raise ValueError(f"invalid quant type '{v}'")+ raise ValueError(f"invalid quant type '{v}'")return v+class IndexMetaConfig(BaseModel):- model_config = ConfigDict(extra='forbid')-+ model_config = ConfigDict(extra="forbid")+IndexName: strVectorIndex: VectorIndexConfigScalarIndex: Optional[List[str]] = None@@ -225,15 +236,17 @@ class IndexMetaConfig(BaseModel):ProjectName: Optional[str] = NoneCollectionName: Optional[str] = None- @field_validator('IndexName', 'ProjectName', 'CollectionName')+ @field_validator("IndexName", "ProjectName", "CollectionName")@classmethoddef validate_names(cls, v):- if v is None: return v+ if v is None:+ return vreturn validate_name_str(v)+class IndexMetaUpdateConfig(BaseModel):- model_config = ConfigDict(extra='forbid')-+ model_config = ConfigDict(extra="forbid")+IndexName: Optional[str] = NoneVectorIndex: Optional[VectorIndexConfig] = NoneScalarIndex: Optional[List[str]] = None@@ -241,31 +254,35 @@ class IndexMetaUpdateConfig(BaseModel):ProjectName: Optional[str] = NoneCollectionName: Optional[str] = None- @field_validator('IndexName', 'ProjectName', 'CollectionName')+ @field_validator("IndexName", "ProjectName", "CollectionName")@classmethoddef validate_names(cls, v):- if v is None: return v+ if v is None:+ return vreturn validate_name_str(v)+class CollectionMetaUpdateConfig(BaseModel):- model_config = ConfigDict(extra='forbid')-+ model_config = ConfigDict(extra="forbid")+CollectionName: Optional[str] = NoneFields: Optional[List[CollectionField]] = NoneProjectName: Optional[str] = NoneDescription: Optional[str] = Field(None, max_length=65535)Vectorize: Optional[VectorizeConfig] = None-- @field_validator('CollectionName', 'ProjectName')++ @field_validator("CollectionName", "ProjectName")@classmethoddef validate_names(cls, v):- if v is None: return v+ if v is None:+ return vreturn validate_name_str(v)- @field_validator('Fields')+ @field_validator("Fields")@classmethoddef validate_fields_list(cls, fields):- if fields is None: return fields+ if fields is None:+ return fieldsnames = set()has_pk = Falsefor f in fields:@@ -278,28 +295,32 @@ class CollectionMetaUpdateConfig(BaseModel):has_pk = Truereturn fields+# --- Helper / Compatibility Functions ---+def _handle_validation_error(e: PydanticValidationError):# Convert Pydantic ValidationError to our custom ValidationError string format- # to maintain some resemblance of old error messages if needed,+ # to maintain some resemblance of old error messages if needed,# or just raise our custom exception.msg = str(e)# Extract first error for cleaner messagetry:err = e.errors()[0]- field = ".".join(str(x) for x in err['loc'])+ field = ".".join(str(x) for x in err["loc"])msg = f"{err['msg']} (field: {field})"except:passraise ValidationError(msg)+def validate_collection_meta_data(meta_data: dict) -> None:try:CollectionMetaConfig.model_validate(meta_data)except PydanticValidationError as e:_handle_validation_error(e)+def is_valid_collection_meta_data(meta_data: dict) -> bool:try:validate_collection_meta_data(meta_data)@@ -307,12 +328,14 @@ def is_valid_collection_meta_data(meta_data: dict) -> bool:except ValidationError:return False+def validate_collection_meta_data_for_update(meta_data: dict, field_meta_dict: dict = None) -> None:try:CollectionMetaUpdateConfig.model_validate(meta_data)except PydanticValidationError as e:_handle_validation_error(e)+def is_valid_collection_meta_data_for_update(meta_data: dict, field_meta_dict: dict = None) -> bool:try:validate_collection_meta_data_for_update(meta_data, field_meta_dict)@@ -320,6 +343,7 @@ def is_valid_collection_meta_data_for_update(meta_data: dict, field_meta_dict: dexcept ValidationError:return False+def validate_index_meta_data(meta_data: dict, field_meta_dict: dict) -> None:try:model = IndexMetaConfig.model_validate(meta_data)@@ -327,10 +351,13 @@ def validate_index_meta_data(meta_data: dict, field_meta_dict: dict) -> None:if model.ScalarIndex:unknown_fields = set(model.ScalarIndex) - set(field_meta_dict.keys())if unknown_fields:- raise ValidationError(f"scalar index contains unknown fields: {list(unknown_fields)}")+ raise ValidationError(+ f"scalar index contains unknown fields: {list(unknown_fields)}"+ )except PydanticValidationError as e:_handle_validation_error(e)+def is_valid_index_meta_data(meta_data: dict, field_meta_dict: dict) -> bool:try:validate_index_meta_data(meta_data, field_meta_dict)@@ -338,16 +365,20 @@ def is_valid_index_meta_data(meta_data: dict, field_meta_dict: dict) -> bool:except ValidationError:return False+def validate_index_meta_data_for_update(meta_data: dict, field_meta_dict: dict) -> None:try:model = IndexMetaUpdateConfig.model_validate(meta_data)if model.ScalarIndex:unknown_fields = set(model.ScalarIndex) - set(field_meta_dict.keys())if unknown_fields:- raise ValidationError(f"scalar index contains unknown fields: {list(unknown_fields)}")+ raise ValidationError(+ f"scalar index contains unknown fields: {list(unknown_fields)}"+ )except PydanticValidationError as e:_handle_validation_error(e)+def is_valid_index_meta_data_for_update(meta_data: dict, field_meta_dict: dict) -> bool:try:validate_index_meta_data_for_update(meta_data, field_meta_dict)@@ -355,6 +386,7 @@ def is_valid_index_meta_data_for_update(meta_data: dict, field_meta_dict: dict)except ValidationError:return False+def fix_collection_meta(meta_data: dict) -> dict:# This logic mutates the input dict to add AUTO_ID if missing, etc.# We can perform this on the dict directly or parse into model, modify, dump.@@ -365,24 +397,27 @@ def fix_collection_meta(meta_data: dict) -> dict:if item.get("IsPrimaryKey", False):has_pk = Truebreak-+if not has_pk:- fields.append({- "FieldName": "AUTO_ID",- "FieldType": "int64",- "IsPrimaryKey": True,- })-+ fields.append(+ {+ "FieldName": "AUTO_ID",+ "FieldType": "int64",+ "IsPrimaryKey": True,+ }+ )+field_count = meta_data.get("_FieldsCount", 0)for item in fields:if "_FieldID" not in item:item["_FieldID"] = field_countfield_count += 1-+meta_data["_FieldsCount"] = field_countmeta_data["Fields"] = fieldsreturn meta_data+# Data validation logic - kept mostly manual or lightweight as creating models for dynamic data row is expensiveREQUIRED_COLLECTION_FIELD_TYPE_CHECK = {"int64": ([int], None, 0),@@ -398,7 +433,7 @@ REQUIRED_COLLECTION_FIELD_TYPE_CHECK = {"vector": ([list],# dim check is done elsewhere or we assume valid if it's a list of floats- lambda l: all(isinstance(item, (int, float)) for item in l),+ lambda l: all(isinstance(item, (int, float)) for item in l),[],),"text": ([str], None, ""),@@ -410,9 +445,12 @@ REQUIRED_COLLECTION_FIELD_TYPE_CHECK = {"sparse_vector": ([dict], None, {}),}+def validate_fields_data(field_data_dict: dict, field_meta_dict: dict) -> None:if len(field_data_dict) > len(field_meta_dict):- raise ValidationError(f"too many fields: got {len(field_data_dict)}, expected max {len(field_meta_dict)}")+ raise ValidationError(+ f"too many fields: got {len(field_data_dict)}, expected max {len(field_meta_dict)}"+ )for field_name, field_value in field_data_dict.items():if field_name not in field_meta_dict:@@ -420,29 +458,33 @@ def validate_fields_data(field_data_dict: dict, field_meta_dict: dict) -> None:field_type = field_meta_dict[field_name]["FieldType"]# Compatibility with enum if using Pydantic model for meta dict- if hasattr(field_type, "value"):+ if hasattr(field_type, "value"):field_type = field_type.value-+if field_type not in REQUIRED_COLLECTION_FIELD_TYPE_CHECK:- # Should be caught by meta validation, but safety check- continue+ # Should be caught by meta validation, but safety check+ continueallowed_types, validator, _ = REQUIRED_COLLECTION_FIELD_TYPE_CHECK[field_type]-+if type(field_value) not in allowed_types:- raise ValidationError(f"field type mismatch for '{field_name}': expected {field_type}, got {type(field_value).__name__}")-+ raise ValidationError(+ f"field type mismatch for '{field_name}': expected {field_type}, got {type(field_value).__name__}"+ )+if validator and not validator(field_value):- raise ValidationError(f"invalid value for field '{field_name}'")+ raise ValidationError(f"invalid value for field '{field_name}'")+def is_valid_fields_data(field_data_dict: dict, field_meta_dict: dict) -> bool:try:validate_fields_data(field_data_dict, field_meta_dict)return True- except ValidationError as e:+ except ValidationError:# print(f"ValidationError {e}") # Reduce noisereturn False+def fix_fields_data(field_data_dict: dict, field_meta_dict: dict) -> dict:if len(field_data_dict) >= len(field_meta_dict):return field_data_dict@@ -457,7 +499,8 @@ def fix_fields_data(field_data_dict: dict, field_meta_dict: dict) -> dict:field_type = field_meta.FieldTypedefault_val = field_meta.DefaultValue- if hasattr(field_type, "value"): field_type = field_type.value+ if hasattr(field_type, "value"):+ field_type = field_type.valueif default_val is not None:field_data_dict[field_name] = default_val@@ -466,4 +509,3 @@ def fix_fields_data(field_data_dict: dict, field_meta_dict: dict) -> dict:else:field_data_dict[field_name] = REQUIRED_COLLECTION_FIELD_TYPE_CHECK[field_type][2]return field_data_dict-diff --git a/openviking/storage/vectordb/vectorize/vectorizer.py b/openviking/storage/vectordb/vectorize/vectorizer.pyindex e82aae62e..222728f65 100644--- a/openviking/storage/vectordb/vectorize/vectorizer.py+++ b/openviking/storage/vectordb/vectorize/vectorizer.py@@ -2,6 +2,7 @@# SPDX-License-Identifier: Apache-2.0from typing import Any, Dict, List, Optional, Tuple, TypedDict+class DenseMeta(TypedDict, total=False):ModelName: strVersion: str@@ -10,14 +11,17 @@ class DenseMeta(TypedDict, total=False):ImageField: strVideoField: str+class SparseMeta(TypedDict, total=False):ModelName: strVersion: str+class VectorizeMeta(TypedDict, total=False):Dense: DenseMetaSparse: SparseMeta+class VectorizerAdapter:"""Adapter for vectorizer to handle data vectorization.@@ -63,7 +67,9 @@ class VectorizerAdapter:"""return self.dim- def vectorize_raw_data(self, raw_data_list: List[Dict[str, Any]]) -> Tuple[List[List[float]], List[Dict[str, float]]]:+ def vectorize_raw_data(+ self, raw_data_list: List[Dict[str, Any]]+ ) -> Tuple[List[List[float]], List[Dict[str, float]]]:"""Vectorize a list of raw data items.Args:diff --git a/openviking/storage/vectordb/vectorize/volcengine_vectorizer.py b/openviking/storage/vectordb/vectorize/volcengine_vectorizer.pyindex 7f5a2a015..62e3349f0 100644--- a/openviking/storage/vectordb/vectorize/volcengine_vectorizer.py+++ b/openviking/storage/vectordb/vectorize/volcengine_vectorizer.py@@ -10,10 +10,7 @@ from volcengine.auth.SignerV4 import SignerV4from volcengine.base.Request import Requestfrom volcengine.Credentials import Credentials-from openviking.storage.vectordb.vectorize.base import (- BaseVectorizer,- VectorizeResult,-)+from openviking.storage.vectordb.vectorize.base import BaseVectorizer, VectorizeResultclass ClientForDataApi:diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.pyindex 5a253f75e..8525ad2c2 100644--- a/openviking/storage/viking_fs.py+++ b/openviking/storage/viking_fs.py@@ -14,10 +14,10 @@ Responsibilities:import asyncioimport json-from pathlib import PurePathfrom dataclasses import dataclass, fieldfrom datetime import datetime-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union+from pathlib import PurePath+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Unionfrom openviking.storage.vikingdb_interface import VikingDBInterfacefrom openviking.utils.logger import get_logger@@ -173,8 +173,8 @@ class VikingFS:path = self._uri_to_path(uri)result = await asyncio.to_thread(self.agfs.grep, path, pattern, True, case_insensitive)# Change file to uri- if result.get('matches', None) is None:- result['matches'] = []+ if result.get("matches", None) is None:+ result["matches"] = []for match in result.get("matches", []):match["uri"] = self._path_to_uri(match.pop("file"))return result@@ -357,8 +357,11 @@ class VikingFS:FindResult"""from openviking.retrieve import (- ContextType, FindResult, HierarchicalRetriever,- QueryPlan, TypedQuery,+ ContextType,+ FindResult,+ HierarchicalRetriever,+ QueryPlan,+ TypedQuery,)from openviking.retrieve.intent_analyzer import IntentAnalyzer@@ -423,7 +426,8 @@ class VikingFS:async def _execute(tq: TypedQuery):return await retriever.retrieve(- tq, limit=limit,+ tq,+ limit=limit,score_threshold=score_threshold,metadata_filter=filter,)@@ -468,9 +472,7 @@ class VikingFS:link_id = next(f"link_{i}" for i in range(1, 10000) if f"link_{i}" not in existing_ids)- entries.append(- RelationEntry(id=link_id, uris=uris, reason=reason)- )+ entries.append(RelationEntry(id=link_id, uris=uris, reason=reason))await self._write_relation_table(from_path, entries)logger.info(f"[VikingFS] Created link: {from_uri} -> {uris}")@@ -544,6 +546,7 @@ class VikingFS:def _infer_context_type(self, uri: str):"""Infer context_type from URI."""from openviking.retrieve import ContextType+if "/memories" in uri:return ContextType.MEMORYelif "/skills" in uri:@@ -590,7 +593,9 @@ class VikingFS:except Exception as e:logger.warning(f"[VikingFS] Failed to delete {uri} from vector store: {e}")- async def _update_vector_store_uris(self, uris: List[str], old_base: str, new_base: str) -> None:+ async def _update_vector_store_uris(+ self, uris: List[str], old_base: str, new_base: str+ ) -> None:"""Update URIs in vector store (when moving files).Preserves vector data, only updates uri and parent_uri fields, no need to regenerate embeddings.@@ -619,12 +624,18 @@ class VikingFS:new_uri = uri.replace(old_base_uri, new_base_uri, 1)old_parent_uri = record.get("parent_uri", "")- new_parent_uri = old_parent_uri.replace(old_base_uri, new_base_uri, 1) if old_parent_uri else ""+ new_parent_uri = (+ old_parent_uri.replace(old_base_uri, new_base_uri, 1) if old_parent_uri else ""+ )- await storage.update("context", record_id, {- "uri": new_uri,- "parent_uri": new_parent_uri,- })+ await storage.update(+ "context",+ record_id,+ {+ "uri": new_uri,+ "parent_uri": new_parent_uri,+ },+ )logger.info(f"[VikingFS] Updated URI: {uri} -> {new_uri}")except Exception as e:logger.warning(f"[VikingFS] Failed to update {uri} in vector store: {e}")@@ -656,12 +667,10 @@ class VikingFS:table_path = f"{dir_path}/.relations.json"try:content = await asyncio.to_thread(self.agfs.read, table_path)- data = json.loads(- content.decode("utf-8") if isinstance(content, bytes) else content- )+ data = json.loads(content.decode("utf-8") if isinstance(content, bytes) else content)except FileNotFoundError:return []- except Exception as e:+ except Exception:# logger.warning(f"[VikingFS] Failed to read relation table {table_path}: {e}")return []@@ -673,8 +682,8 @@ class VikingFS:entries.append(RelationEntry.from_dict(entry_data))elif isinstance(data, dict):# Old format: nested {namespace: {user: [entries]}}- for namespace, user_dict in data.items():- for user, entry_list in user_dict.items():+ for _namespace, user_dict in data.items():+ for _user, entry_list in user_dict.items():for entry_data in entry_list:entries.append(RelationEntry.from_dict(entry_data))return entries@@ -811,6 +820,7 @@ class VikingFS:def create_temp_uri(self) -> str:"""Create temp directory URI."""import uuid+temp_id = uuid.uuid4().hex[:8]return f"viking://temp/{temp_id}"diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.pyindex f67a3e200..87b300bc0 100644--- a/openviking/storage/viking_vector_index_backend.py+++ b/openviking/storage/viking_vector_index_backend.py@@ -9,15 +9,12 @@ Supports both in-memory and local persistent storage modes.import uuidfrom pathlib import Path-from typing import Any, Dict, List, Optional, Tuple, Union+from typing import Any, Dict, List, Optional, Tuple-from openviking.storage.vikingdb_interface import (- CollectionNotFoundError,- VikingDBInterface,-)from openviking.storage.vectordb.collection.collection import Collectionfrom openviking.storage.vectordb.collection.result import FetchDataInCollectionResultfrom openviking.storage.vectordb.utils.logging_init import init_cpp_logging+from openviking.storage.vikingdb_interface import CollectionNotFoundError, VikingDBInterfacefrom openviking.utils import get_loggerfrom openviking.utils.config.vectordb_config import VectorDBBackendConfig@@ -82,25 +79,23 @@ class VikingVectorIndexBackend(VikingDBInterface):self.vector_dim = config.vector_dimself.distance_metric = config.distance_metric-+if config.backend == "volcengine":if not (- config.volcengine+ config.volcengineand config.volcengine.akand config.volcengine.skand config.volcengine.region):- raise ValueError(- "Volcengine backend requires AK, SK, and Region configuration"- )-+ raise ValueError("Volcengine backend requires AK, SK, and Region configuration")+# Volcengine VikingDB modeself._mode = config.backend# Convert lowercase keys to uppercase for consistency with volcengine_collectionvolc_config = {- "AK": config.volcengine.ak,- "SK": config.volcengine.sk,- "Region": config.volcengine.region,+ "AK": config.volcengine.ak,+ "SK": config.volcengine.sk,+ "Region": config.volcengine.region,}from openviking.storage.vectordb.project.volcengine_project import (@@ -239,7 +234,11 @@ class VikingVectorIndexBackend(VikingDBInterface):is_primary_key = field.get("IsPrimaryKey", False)# Index all non-vector, non-primary-key, and non-date_time fields by default# Volcengine VikingDB doesn't support indexing date_time fields- if field_name and field_type not in ("vector", "sparse_vector", "date_time") and not is_primary_key:+ if (+ field_name+ and field_type not in ("vector", "sparse_vector", "date_time")+ and not is_primary_key+ ):scalar_index_fields.append(field_name)# Create default index for the collection@@ -308,9 +307,7 @@ class VikingVectorIndexBackend(VikingDBInterface):if not self.project.has_collection(name):return None- collection = self.project.get_collection(name)config = self._collection_configs.get(name, {})- meta_data = self._get_meta_data(name, collection)return {"name": name,@@ -354,7 +351,7 @@ class VikingVectorIndexBackend(VikingDBInterface):new_data[k] = data[k]try:- result = coll.upsert_data([new_data])+ coll.upsert_data([new_data])return record_idexcept Exception as e:logger.error(f"Error inserting record: {e}")@@ -445,22 +442,22 @@ class VikingVectorIndexBackend(VikingDBInterface):result = coll.search_by_random(index_name=self.DEFAULT_INDEX_NAME,limit=10,- filters={"op": "must", "field": "uri", "conds": [uri]}+ filters={"op": "must", "field": "uri", "conds": [uri]},)records = []for item in result.data:record = dict(item.fields) if item.fields else {}record["id"] = item.idrecords.append(record)- if (len(records) > 0):+ if len(records) > 0:raise ValueError(f"Duplicate records found for URI: {uri}")- if (len(records) == 0):+ if len(records) == 0:raise ValueError(f"Record not found for URI: {uri}")return records[0]except Exception as e:logger.error(f"Error fetching record by URI '{uri}': {e}")return None-+async def exists(self, collection: str, id: str) -> bool:"""Check if a record exists."""try:@@ -842,7 +839,7 @@ class VikingVectorIndexBackend(VikingDBInterface):"""Check if storage backend is healthy and accessible."""try:# Simple check: verify we can access the project- collections = self.project.list_collections()+ self.project.list_collections()return Trueexcept Exception:return False@@ -858,9 +855,7 @@ class VikingVectorIndexBackend(VikingDBInterface):try:coll = self._get_collection(collection_name)result = coll.aggregate_data(- index_name=self.DEFAULT_INDEX_NAME,- op="count",- filters=None+ index_name=self.DEFAULT_INDEX_NAME, op="count", filters=None)total_records += result.agg.get("_total", 0)except Exception as e:diff --git a/openviking/storage/vikingdb_manager.py b/openviking/storage/vikingdb_manager.pyindex 8ecfb8896..af38c9e6f 100644--- a/openviking/storage/vikingdb_manager.py+++ b/openviking/storage/vikingdb_manager.py@@ -4,13 +4,13 @@VikingDB Manager class that extends VikingVectorIndexBackend with queue management functionality."""-from typing import Any, Dict, List, Optional, Union-from pathlib import Path-from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend-from openviking.storage.queuefs.queue_manager import init_queue_manager+from typing import Optional+from openviking.storage.collection_schemas import TextEmbeddingHandlerfrom openviking.storage.queuefs.embedding_msg import EmbeddingMsgfrom openviking.storage.queuefs.embedding_queue import EmbeddingQueue+from openviking.storage.queuefs.queue_manager import init_queue_manager+from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackendfrom openviking.utils import get_loggerfrom openviking.utils.config.agfs_config import AGFSConfigfrom openviking.utils.config.vectordb_config import VectorDBBackendConfig@@ -86,12 +86,12 @@ class VikingDBManager(VikingVectorIndexBackend):self._embedding_handler = TextEmbeddingHandler(self)# Get embedding queue with the handler, allow creation if not exists- embedding_queue = self._queue_manager.get_queue(+ self._queue_manager.get_queue(self._queue_manager.EMBEDDING,dequeue_handler=self._embedding_handler,allow_create=True,)- logger.info(f"Embedding queue initialized with TextEmbeddingHandler")+ logger.info("Embedding queue initialized with TextEmbeddingHandler")def _init_semantic_queue(self):"""Initialize semantic queue with SemanticProcessor, semantic queue is used to get abstract and summary of context data."""@@ -105,12 +105,12 @@ class VikingDBManager(VikingVectorIndexBackend):self._semantic_processor = SemanticProcessor()# Get semantic queue with the handler, allow creation if not exists- semantic_queue = self._queue_manager.get_queue(+ self._queue_manager.get_queue(self._queue_manager.SEMANTIC,dequeue_handler=self._semantic_processor,allow_create=True,)- logger.info(f"Semantic queue initialized with SemanticProcessor")+ logger.info("Semantic queue initialized with SemanticProcessor")async def close(self) -> None:"""Close storage connection and release resources, including queue manager."""diff --git a/openviking/sync_client.py b/openviking/sync_client.pyindex 05d4316ce..fbe61b70f 100644--- a/openviking/sync_client.py+++ b/openviking/sync_client.py@@ -5,10 +5,9 @@ Synchronous OpenViking client implementation."""import asyncio-from typing import Any, Dict, List, Optional, TYPE_CHECKING+from typing import TYPE_CHECKING, Any, Dict, List, Optionalif TYPE_CHECKING:- from openviking.retrieve import QueryResultfrom openviking.session import Sessionfrom openviking.async_client import AsyncOpenViking@@ -66,9 +65,17 @@ class SyncOpenViking:filter: Optional[Dict] = None,):"""Execute complex retrieval (intent analysis, hierarchical retrieval)."""- return asyncio.run(self._async_client.search(query, target_uri, session, limit, score_threshold, filter))+ return asyncio.run(+ self._async_client.search(query, target_uri, session, limit, score_threshold, filter)+ )- def find(self, query: str, target_uri: str = "", limit: int = 10, score_threshold: Optional[float] = None):+ def find(+ self,+ query: str,+ target_uri: str = "",+ limit: int = 10,+ score_threshold: Optional[float] = None,+ ):"""Quick retrieval"""return asyncio.run(self._async_client.find(query, target_uri, limit, score_threshold))@@ -140,7 +147,7 @@ class SyncOpenViking:def mv(self, from_uri: str, to_uri: str) -> None:"""Move resource"""return asyncio.run(self._async_client.mv(from_uri, to_uri))-+def tree(self, uri: str) -> Dict:"""Get directory tree"""return asyncio.run(self._async_client.tree(uri))diff --git a/openviking/utils/config/__init__.py b/openviking/utils/config/__init__.pyindex bac6cf272..e6a811413 100644--- a/openviking/utils/config/__init__.py+++ b/openviking/utils/config/__init__.py@@ -1,33 +1,32 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0+from .agfs_config import AGFSConfigfrom .embedding_config import EmbeddingConfigfrom .open_viking_config import (OpenVikingConfig,OpenVikingConfigSingleton,get_openviking_config,- set_openviking_config,is_valid_openviking_config,+ set_openviking_config,)-from .rerank_config import RerankConfig-from .storage_config import StorageConfig-from .agfs_config import AGFSConfig-from .vectordb_config import VectorDBBackendConfig-from .vlm_config import VLMConfigfrom .parser_config import (- ParserConfig,- PDFConfig,+ PARSER_CONFIG_REGISTRY,+ AudioConfig,CodeConfig,+ HTMLConfig,ImageConfig,- AudioConfig,- VideoConfig,MarkdownConfig,- HTMLConfig,+ ParserConfig,+ PDFConfig,TextConfig,+ VideoConfig,get_parser_config,load_parser_configs_from_dict,- PARSER_CONFIG_REGISTRY,)-+from .rerank_config import RerankConfig+from .storage_config import StorageConfig+from .vectordb_config import VectorDBBackendConfig+from .vlm_config import VLMConfig__all__ = ["AGFSConfig",diff --git a/openviking/utils/config/agfs_config.py b/openviking/utils/config/agfs_config.pyindex 7d3f12f76..166f1ae40 100644--- a/openviking/utils/config/agfs_config.py+++ b/openviking/utils/config/agfs_config.py@@ -14,7 +14,9 @@ class AGFSConfig(BaseModel):log_level: str = Field(default="warn", description="AGFS log level")- url: Optional[str] = Field(default="http://localhost:8080", description="AGFS service URL for service mode")+ url: Optional[str] = Field(+ default="http://localhost:8080", description="AGFS service URL for service mode"+ )backend: str = Field(default="local", description="AGFS storage backend: 'local' | 's3' | 'memory'"@@ -33,7 +35,7 @@ class AGFSConfig(BaseModel):s3_secret_key: Optional[str] = Field(default=None, description="S3 secret key")- @model_validator(mode='after')+ @model_validator(mode="after")def validate_config(self):"""Validate configuration completeness and consistency"""if self.backend not in ["local", "s3", "memory"]:diff --git a/openviking/utils/config/embedding_config.py b/openviking/utils/config/embedding_config.pyindex b43be6d52..ae3ef8c7b 100644--- a/openviking/utils/config/embedding_config.py+++ b/openviking/utils/config/embedding_config.py@@ -1,39 +1,45 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0import os-from typing import List, Optional, Union, Dict, Any-import openai-import volcenginesdkarkruntime+from typing import Any, Optional+from pydantic import BaseModel, Field, model_validator+class EmbeddingModelConfig(BaseModel):"""Configuration for a specific embedding model"""+model: Optional[str] = Field(default=None, description="Model name")api_key: Optional[str] = Field(default=None, description="API key")api_base: Optional[str] = Field(default=None, description="API base URL")dimension: Optional[int] = Field(default=None, description="Embedding dimension")batch_size: int = Field(default=32, description="Batch size for embedding generation")input: str = Field(default="multimodal", description="Input type: 'text' or 'multimodal'")- provider: Optional[str] = Field(default="volcengine", description="Provider type: 'openai', 'volcengine', 'vikingdb'")- backend: Optional[str] = Field(default="volcengine", description="Backend type (Deprecated, use 'provider' instead): 'openai', 'volcengine', 'vikingdb'")+ provider: Optional[str] = Field(+ default="volcengine", description="Provider type: 'openai', 'volcengine', 'vikingdb'"+ )+ backend: Optional[str] = Field(+ default="volcengine",+ description="Backend type (Deprecated, use 'provider' instead): 'openai', 'volcengine', 'vikingdb'",+ )version: Optional[str] = Field(default=None, description="Model version")ak: Optional[str] = Field(default=None, description="Access Key ID for VikingDB API")sk: Optional[str] = Field(default=None, description="Access Key Secretfor VikingDB API")region: Optional[str] = Field(default=None, description="Region for VikingDB API")host: Optional[str] = Field(default=None, description="Host for VikingDB API")- @model_validator(mode='before')+ @model_validator(mode="before")@classmethoddef sync_provider_backend(cls, data: Any) -> Any:if isinstance(data, dict):- provider = data.get('provider')- backend = data.get('backend')-+ provider = data.get("provider")+ backend = data.get("backend")+if backend is not None and provider is None:- data['provider'] = backend+ data["provider"] = backendreturn data- @model_validator(mode='after')+ @model_validator(mode="after")def validate_config(self):"""Validate configuration completeness and consistency"""if self.backend and not self.provider:@@ -101,14 +107,23 @@ class EmbeddingConfig(BaseModel):config_sections = [("dense", "OPENVIKING_EMBEDDING_DENSE_"),("sparse", "OPENVIKING_EMBEDDING_SPARSE_"),- ("hybrid", "OPENVIKING_EMBEDDING_HYBRID_")+ ("hybrid", "OPENVIKING_EMBEDDING_HYBRID_"),]env_fields = {- "model": str, "api_key": str, "api_base": str,- "dimension": int, "batch_size": int,- "input": str, "provider": str, "backend": str, "version": str,- "ak": str, "sk": str, "region": str, "host": str+ "model": str,+ "api_key": str,+ "api_base": str,+ "dimension": int,+ "batch_size": int,+ "input": str,+ "provider": str,+ "backend": str,+ "version": str,+ "ak": str,+ "sk": str,+ "region": str,+ "host": str,}for section, prefix in config_sections:@@ -133,7 +148,7 @@ class EmbeddingConfig(BaseModel):return data- @model_validator(mode='after')+ @model_validator(mode="after")def validate_config(self):"""Validate configuration completeness and consistency"""if not self.dense and not self.sparse and not self.hybrid:@@ -158,89 +173,89 @@ class EmbeddingConfig(BaseModel):"""from openviking.models.embedder import (OpenAIDenseEmbedder,- VolcengineDenseEmbedder,- VolcengineSparseEmbedder,- VolcengineHybridEmbedder,VikingDBDenseEmbedder,+ VikingDBHybridEmbedder,VikingDBSparseEmbedder,- VikingDBHybridEmbedder+ VolcengineDenseEmbedder,+ VolcengineHybridEmbedder,+ VolcengineSparseEmbedder,)# Factory registry: (provider, type) -> (embedder_class, param_builder)factory_registry = {- ('openai', 'dense'): (+ ("openai", "dense"): (OpenAIDenseEmbedder,lambda cfg: {- 'model_name': cfg.model,- 'api_key': cfg.api_key,- 'api_base': cfg.api_base,- 'dimension': cfg.dimension- }+ "model_name": cfg.model,+ "api_key": cfg.api_key,+ "api_base": cfg.api_base,+ "dimension": cfg.dimension,+ },),- ('volcengine', 'dense'): (+ ("volcengine", "dense"): (VolcengineDenseEmbedder,lambda cfg: {- 'model_name': cfg.model,- 'api_key': cfg.api_key,- 'api_base': cfg.api_base,- 'dimension': cfg.dimension,- 'input_type': cfg.input- }+ "model_name": cfg.model,+ "api_key": cfg.api_key,+ "api_base": cfg.api_base,+ "dimension": cfg.dimension,+ "input_type": cfg.input,+ },),- ('volcengine', 'sparse'): (+ ("volcengine", "sparse"): (VolcengineSparseEmbedder,lambda cfg: {- 'model_name': cfg.model,- 'api_key': cfg.api_key,- 'api_base': cfg.api_base- }+ "model_name": cfg.model,+ "api_key": cfg.api_key,+ "api_base": cfg.api_base,+ },),- ('volcengine', 'hybrid'): (+ ("volcengine", "hybrid"): (VolcengineHybridEmbedder,lambda cfg: {- 'model_name': cfg.model,- 'api_key': cfg.api_key,- 'api_base': cfg.api_base,- 'dimension': cfg.dimension,- 'input_type': cfg.input- }+ "model_name": cfg.model,+ "api_key": cfg.api_key,+ "api_base": cfg.api_base,+ "dimension": cfg.dimension,+ "input_type": cfg.input,+ },),- ('vikingdb', 'dense'): (+ ("vikingdb", "dense"): (VikingDBDenseEmbedder,lambda cfg: {- 'model_name': cfg.model,- 'model_version': cfg.version,- 'ak': cfg.ak,- 'sk': cfg.sk,- 'region': cfg.region,- 'host': cfg.host,- 'dimension': cfg.dimension,- 'input_type': cfg.input- }+ "model_name": cfg.model,+ "model_version": cfg.version,+ "ak": cfg.ak,+ "sk": cfg.sk,+ "region": cfg.region,+ "host": cfg.host,+ "dimension": cfg.dimension,+ "input_type": cfg.input,+ },),- ('vikingdb', 'sparse'): (+ ("vikingdb", "sparse"): (VikingDBSparseEmbedder,lambda cfg: {- 'model_name': cfg.model,- 'model_version': cfg.version,- 'ak': cfg.ak,- 'sk': cfg.sk,- 'region': cfg.region,- 'host': cfg.host- }+ "model_name": cfg.model,+ "model_version": cfg.version,+ "ak": cfg.ak,+ "sk": cfg.sk,+ "region": cfg.region,+ "host": cfg.host,+ },),- ('vikingdb', 'hybrid'): (+ ("vikingdb", "hybrid"): (VikingDBHybridEmbedder,lambda cfg: {- 'model_name': cfg.model,- 'model_version': cfg.version,- 'ak': cfg.ak,- 'sk': cfg.sk,- 'region': cfg.region,- 'host': cfg.host,- 'dimension': cfg.dimension,- 'input_type': cfg.input- }+ "model_name": cfg.model,+ "model_version": cfg.version,+ "ak": cfg.ak,+ "sk": cfg.sk,+ "region": cfg.region,+ "host": cfg.host,+ "dimension": cfg.dimension,+ "input_type": cfg.input,+ },),}@@ -267,15 +282,17 @@ class EmbeddingConfig(BaseModel):from openviking.models.embedder import CompositeHybridEmbedderif self.hybrid:- return self._create_embedder(self.hybrid.provider.lower(), 'hybrid', self.hybrid)+ return self._create_embedder(self.hybrid.provider.lower(), "hybrid", self.hybrid)if self.dense and self.sparse:- dense_embedder = self._create_embedder(self.dense.provider.lower(), 'dense', self.dense)- sparse_embedder = self._create_embedder(self.sparse.provider.lower(), 'sparse', self.sparse)+ dense_embedder = self._create_embedder(self.dense.provider.lower(), "dense", self.dense)+ sparse_embedder = self._create_embedder(+ self.sparse.provider.lower(), "sparse", self.sparse+ )return CompositeHybridEmbedder(dense_embedder, sparse_embedder)if self.dense:- return self._create_embedder(self.dense.provider.lower(), 'dense', self.dense)+ return self._create_embedder(self.dense.provider.lower(), "dense", self.dense)raise ValueError("No embedding configuration found (dense, sparse, or hybrid)")@@ -291,4 +308,3 @@ class EmbeddingConfig(BaseModel):if self.dense:return self.dense.dimension or 2048return 2048-diff --git a/openviking/utils/config/open_viking_config.py b/openviking/utils/config/open_viking_config.pyindex 5b0aea451..fe5c17051 100644--- a/openviking/utils/config/open_viking_config.py+++ b/openviking/utils/config/open_viking_config.py@@ -1,29 +1,27 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0import json-import loggingimport osfrom pathlib import Path-from typing import Any, Dict, Optionalfrom threading import Lock+from typing import Any, Dict, Optionalfrom pydantic import BaseModel, Fieldfrom .embedding_config import EmbeddingConfig-from .rerank_config import RerankConfig-from .storage_config import StorageConfig-from .vlm_config import VLMConfigfrom .parser_config import (- PDFConfig,+ AudioConfig,CodeConfig,+ HTMLConfig,ImageConfig,- AudioConfig,- VideoConfig,MarkdownConfig,- HTMLConfig,+ PDFConfig,TextConfig,- load_parser_configs_from_dict,+ VideoConfig,)+from .rerank_config import RerankConfig+from .storage_config import StorageConfig+from .vlm_config import VLMConfigclass OpenVikingConfig(BaseModel):@@ -246,7 +244,9 @@ def is_valid_openviking_config(config: OpenVikingConfig) -> bool:)if errors:- error_message = "Invalid OpenViking configuration:\n" + "\n".join(f" - {e}" for e in errors)+ error_message = "Invalid OpenViking configuration:\n" + "\n".join(+ f" - {e}" for e in errors+ )raise ValueError(error_message)return Truediff --git a/openviking/utils/config/parser_config.py b/openviking/utils/config/parser_config.pyindex 3e3ba0343..fbfb794d9 100644--- a/openviking/utils/config/parser_config.py+++ b/openviking/utils/config/parser_config.py@@ -8,10 +8,10 @@ scattered across different modules. All configurations inherit from ParserConfigand can be loaded from ov.conf files."""-from dataclasses import dataclass, field-from pathlib import Path-from typing import Any, Dict, List, Optional, Unionimport os+from dataclasses import dataclass+from pathlib import Path+from typing import Any, Dict, Optional, Union@dataclass@@ -111,11 +111,11 @@ class ParserConfig:value = os.environ[env_var]# Type conversionfield_type = cls.__dataclass_fields__[field_name].type- if field_type == bool:+ if field_type is bool:data[field_name] = value.lower() in ("true", "1", "yes")- elif field_type == int:+ elif field_type is int:data[field_name] = int(value)- elif field_type == float:+ elif field_type is float:data[field_name] = float(value)else:data[field_name] = valuediff --git a/openviking/utils/config/storage_config.py b/openviking/utils/config/storage_config.pyindex cfd6d8731..c9661012c 100644--- a/openviking/utils/config/storage_config.py+++ b/openviking/utils/config/storage_config.py@@ -1,17 +1,17 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0-from typing import Any, Dict, Optional+from typing import Any, Dictfrom pydantic import BaseModel, Fieldfrom .agfs_config import AGFSConfigfrom .vectordb_config import VectorDBBackendConfig+class StorageConfig(BaseModel):"""Configuration for storage backend."""- agfs: AGFSConfig = Field(- default_factory=lambda: AGFSConfig(), description="AGFS configuration"- )++ agfs: AGFSConfig = Field(default_factory=lambda: AGFSConfig(), description="AGFS configuration")vectordb: VectorDBBackendConfig = Field(default_factory=lambda: VectorDBBackendConfig(),diff --git a/openviking/utils/config/vectordb_config.py b/openviking/utils/config/vectordb_config.pyindex 85b7587ec..dc32c7e42 100644--- a/openviking/utils/config/vectordb_config.py+++ b/openviking/utils/config/vectordb_config.py@@ -1,11 +1,13 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0-from typing import Any, Dict, Optional+from typing import Optionalfrom pydantic import BaseModel, Field, model_validator+class VolcengineConfig(BaseModel):"""Configuration for Volcengine VikingDB."""+ak: Optional[str] = Field(default=None, description="Volcengine Access Key")sk: Optional[str] = Field(default=None, description="Volcengine Secret Key")region: Optional[str] = Field(@@ -13,6 +15,7 @@ class VolcengineConfig(BaseModel):)host: Optional[str] = Field(default=None, description="Volcengine VikingDB host (optional)")+class VectorDBBackendConfig(BaseModel):"""Configuration for VectorDB backend.@@ -20,6 +23,7 @@ class VectorDBBackendConfig(BaseModel):This configuration class consolidates all settings related to the VectorDB backend,including type, connection details, and backend-specific parameters."""+backend: str = Field(default="local",description="VectorDB backend type: 'local' (file-based), 'http' (remote service), or 'volcengine' (VikingDB)",@@ -49,7 +53,7 @@ class VectorDBBackendConfig(BaseModel):description="Volcengine VikingDB configuration for 'volcengine' type",)- @model_validator(mode='after')+ @model_validator(mode="after")def validate_config(self):"""Validate configuration completeness and consistency"""if self.backend not in ["local", "http", "volcengine"]:@@ -71,4 +75,4 @@ class VectorDBBackendConfig(BaseModel):if not self.volcengine.region:raise ValueError("VectorDB volcengine backend requires 'region' to be set")- return self\ No newline at end of file+ return selfdiff --git a/openviking/utils/config/vlm_config.py b/openviking/utils/config/vlm_config.pyindex 039c4f945..6511b9e68 100644--- a/openviking/utils/config/vlm_config.py+++ b/openviking/utils/config/vlm_config.py@@ -1,11 +1,11 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0import os-from typing import Any, Dict, Literal, Optional+from typing import Any, Literal, Optionalfrom pydantic import BaseModel, Field, model_validator-from openviking.models.vlm import VLMFactory, VLMBase+from openviking.models.vlm import VLMBase, VLMFactoryclass VLMConfig(BaseModel):@@ -28,15 +28,15 @@ class VLMConfig(BaseModel):class Config:arbitrary_types_allowed = True- @model_validator(mode='before')+ @model_validator(mode="before")@classmethoddef sync_provider_backend(cls, data: Any) -> Any:if isinstance(data, dict):- provider = data.get('provider')- backend = data.get('backend')-+ provider = data.get("provider")+ backend = data.get("backend")+if backend is not None and provider is None:- data['provider'] = backend+ data["provider"] = backendreturn data@model_validator(mode="before")@@ -58,7 +58,7 @@ class VLMConfig(BaseModel):data[field] = env_valreturn data- @model_validator(mode='after')+ @model_validator(mode="after")def validate_config(self):"""Validate configuration completeness and consistency"""if self.backend and not self.provider:diff --git a/openviking/utils/llm.py b/openviking/utils/llm.pyindex 5919afca5..d865a9d8e 100644--- a/openviking/utils/llm.py+++ b/openviking/utils/llm.py@@ -10,16 +10,16 @@ import jsonimport refrom typing import Any, Dict, Optional, Type, TypeVar+import json_repairfrom pydantic import BaseModelfrom openviking.utils.logger import get_logger-import json_repairlogger = get_logger(__name__)T = TypeVar("T", bound=BaseModel)-+def parse_json_from_response(response: str) -> Optional[Any]:"""Parse JSON object from LLM text response.@@ -78,6 +78,7 @@ def parse_json_from_response(response: str) -> Optional[Any]:return None+def _fix_json_quotes(json_str: str) -> str:"""Fix quote issues in JSON string@@ -106,6 +107,7 @@ def _fix_json_quotes(json_str: str) -> str:except:return json_str+def parse_json_to_model(response: str, model_class: Type[T]) -> Optional[T]:"""Parse JSON response into a Pydantic model.@@ -170,6 +172,7 @@ class StructuredLLM:def _get_vlm(self):"""Get VLM singleton."""from openviking.utils.config import get_openviking_config+return get_openviking_config().vlmdef complete_json(diff --git a/openviking/utils/media_processor.py b/openviking/utils/media_processor.pyindex 9efdc5560..39de9ae2a 100644--- a/openviking/utils/media_processor.py+++ b/openviking/utils/media_processor.py@@ -31,6 +31,7 @@ class UnifiedResourceProcessor:def _get_vlm_processor(self) -> Optional["VLMProcessor"]:if self._vlm_processor is None:from openviking.parse.vlm import VLMProcessor+self._vlm_processor = VLMProcessor()return self._vlm_processordiff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.pyindex 1b3bcc5a7..24e28cc53 100644--- a/openviking/utils/resource_processor.py+++ b/openviking/utils/resource_processor.py@@ -7,17 +7,17 @@ Handles coordinated writes and self-iteration processesas described in the OpenViking design document."""-from pathlib import Path-from typing import Any, Dict, List, Optional+from typing import TYPE_CHECKING, Any, Dict, Optionalfrom openviking.parse.tree_builder import TreeBuilder-from openviking.utils.extractor import MediaStrategyfrom openviking.storage import VikingDBManagerfrom openviking.storage.viking_fs import get_viking_fs-from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverterfrom openviking.utils import get_loggerfrom openviking.utils.storage import StoragePath+if TYPE_CHECKING:+ from openviking.parse.vlm import VLMProcessor+logger = get_logger(__name__)diff --git a/openviking/utils/skill_processor.py b/openviking/utils/skill_processor.pyindex a705403c7..b84a79f34 100644--- a/openviking/utils/skill_processor.py+++ b/openviking/utils/skill_processor.py@@ -6,13 +6,12 @@ Skill Processor for OpenViking.Handles skill parsing, LLM generation, and storage operations."""-from datetime import datetimefrom pathlib import Pathfrom typing import Any, Dict, List, Optionalfrom openviking.core.context import Context, ContextType, Vectorize+from openviking.core.mcp_converter import is_mcp_format, mcp_to_skillfrom openviking.core.skill_loader import SkillLoader-from openviking.core.mcp_converter import mcp_to_skill, is_mcp_formatfrom openviking.storage import VikingDBManagerfrom openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverterfrom openviking.storage.viking_fs import VikingFS@@ -72,7 +71,7 @@ class SkillProcessor:"allowed_tools": skill_dict.get("allowed_tools", []),"tags": skill_dict.get("tags", []),"source_path": skill_dict.get("source_path", ""),- }+ },)context.set_vectorize(Vectorize(text=context.abstract))diff --git a/openviking/utils/uri.py b/openviking/utils/uri.pyindex 678ae4d53..1c994148a 100644--- a/openviking/utils/uri.py+++ b/openviking/utils/uri.py@@ -108,7 +108,7 @@ class VikingURI:True if matches, False otherwise"""return self.uri.startswith(prefix)-+@propertydef parent(self) -> Optional["VikingURI"]:"""@@ -126,7 +126,7 @@ class VikingURI:if scheme_end == -1:return None- after_scheme = uri[scheme_end + len(scheme_sep):]+ after_scheme = uri[scheme_end + len(scheme_sep) :]# If no / in after_scheme, only scope exists, no pathif "/" not in after_scheme:diff --git a/pyproject.toml b/pyproject.tomlindex 8e3f16497..9cf093a22 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -41,8 +41,6 @@ dependencies = ["pytest>=7.0.0","pytest-asyncio>=0.21.0","pytest-cov>=4.0.0",- "black>=23.0.0",- "isort>=5.12.0","mypy>=1.0.0","ruff>=0.1.0","sphinx>=7.0.0",@@ -78,15 +76,6 @@ exclude = ["tests*", "docs*", "examples*"][tool.setuptools.package-data]openviking = ["prompts/templates/**/*.yaml"]-[tool.black]-line-length = 100-target-version = ["py39", "py310", "py311", "py312"]-include = '\.pyi?$'--[tool.isort]-profile = "black"-line_length = 100-[tool.mypy]python_version = "3.9"warn_return_any = false@@ -109,6 +98,8 @@ addopts = "-v --cov=openviking --cov-report=term-missing"[tool.ruff]line-length = 100+exclude = ["third_party"]+target-version = "py39"[tool.ruff.lint]select = [@@ -132,3 +123,10 @@ ignore = [[tool.ruff.lint.per-file-ignores]"__init__.py" = ["F401"] # Allow unused imports in __init__.py++[tool.ruff.format]+quote-style = "double"+indent-style = "space"+skip-magic-trailing-comma = false+line-ending = "auto"+diff --git a/setup.py b/setup.pyindex e1fba9d47..cc72f84ce 100644--- a/setup.py+++ b/setup.py@@ -1,8 +1,11 @@import os-import sysimport shutil+import sys+import sysconfigfrom pathlib import Path-from setuptools import setup, find_packages, Extension++import pybind11+from setuptools import Extension, find_packages, setupfrom setuptools.command.build_ext import build_extCMAKE_PATH = shutil.which("cmake") or "cmake"@@ -34,7 +37,7 @@ class CMakeBuildExtension(build_ext):# Pathsbinary_name = "agfs-server.exe" if sys.platform == "win32" else "agfs-server"agfs_server_dir = Path("third_party/agfs/agfs-server").resolve()-+# Target in source tree (for development/install)agfs_bin_dir = Path("openviking/bin").resolve()agfs_target_binary = agfs_bin_dir / binary_name@@ -43,34 +46,47 @@ class CMakeBuildExtension(build_ext):if agfs_server_dir.exists() and shutil.which("go"):print("Building AGFS server from source...")import subprocess+try:- build_args = ["go", "build", "-o", f"build/{binary_name}", "cmd/server/main.go"] if sys.platform == "win32" else ["make", "build"]-+ build_args = (+ ["go", "build", "-o", f"build/{binary_name}", "cmd/server/main.go"]+ if sys.platform == "win32"+ else ["make", "build"]+ )+subprocess.run(build_args,cwd=str(agfs_server_dir),check=True,stdout=subprocess.PIPE,- stderr=subprocess.PIPE+ stderr=subprocess.PIPE,)agfs_built_binary = agfs_server_dir / "build" / binary_nameif agfs_built_binary.exists():self._copy_binary(agfs_built_binary, agfs_target_binary)- print(f"[OK] AGFS server built successfully from source")+ print("[OK] AGFS server built successfully from source")else:- raise FileNotFoundError(f"Build succeeded but binary not found at {agfs_built_binary}")+ raise FileNotFoundError(+ f"Build succeeded but binary not found at {agfs_built_binary}"+ )except (subprocess.CalledProcessError, Exception) as e:error_msg = f"Failed to build AGFS from source: {e}"if isinstance(e, subprocess.CalledProcessError):- if e.stdout: error_msg += f"\nBuild stdout:\n{e.stdout.decode('utf-8', errors='replace')}"- if e.stderr: error_msg += f"\nBuild stderr:\n{e.stderr.decode('utf-8', errors='replace')}"+ if e.stdout:+ error_msg += (+ f"\nBuild stdout:\n{e.stdout.decode('utf-8', errors='replace')}"+ )+ if e.stderr:+ error_msg += (+ f"\nBuild stderr:\n{e.stderr.decode('utf-8', errors='replace')}"+ )raise RuntimeError(error_msg)else:if not agfs_server_dir.exists():- raise FileNotFoundError(f"AGFS source directory not found at {agfs_server_dir}")+ raise FileNotFoundError(f"AGFS source directory not found at {agfs_server_dir}")else:- raise RuntimeError("Go compiler not found. Please install Go to build AGFS server.")+ raise RuntimeError("Go compiler not found. Please install Go to build AGFS server.")# 2. Ensure AGFS binary is copied to the build directory (where wheel is packaged from)if self.build_lib:@@ -89,7 +105,7 @@ class CMakeBuildExtension(build_ext):cmake_args = [f"-S{Path(ENGINE_SOURCE_DIR).resolve()}",f"-B{build_dir}",- f"-DCMAKE_BUILD_TYPE=Release",+ "-DCMAKE_BUILD_TYPE=Release",f"-DPY_OUTPUT_DIR={ext_dir}","-DCMAKE_VERBOSE_MAKEFILE=ON","-DCMAKE_INSTALL_RPATH=$ORIGIN",@@ -108,17 +124,10 @@ class CMakeBuildExtension(build_ext):self.spawn([self.cmake_executable] + cmake_args)- build_args = [- "--build", str(build_dir),- "--config", "Release",- f"-j{os.cpu_count() or 4}"- ]+ build_args = ["--build", str(build_dir), "--config", "Release", f"-j{os.cpu_count() or 4}"]self.spawn([self.cmake_executable] + build_args)-import sysconfig-import pybind11-setup(ext_modules=[Extension(diff --git a/tests/client/test_file_operations.py b/tests/client/test_file_operations.pyindex d5dd0b741..99415f205 100644--- a/tests/client/test_file_operations.py+++ b/tests/client/test_file_operations.py@@ -24,15 +24,12 @@ class TestRm:uris = await client.tree(result["root_uri"])for data in uris:- if not data['isDir']:- await client.rm(data['uri'])- with pytest.raises(Exception):- await client.read(data['uri'])--- async def test_rm_directory_recursive(- self, client: AsyncOpenViking, sample_directory: Path- ):+ if not data["isDir"]:+ await client.rm(data["uri"])+ with pytest.raises(Exception): # noqa: B017+ await client.read(data["uri"])++ async def test_rm_directory_recursive(self, client: AsyncOpenViking, sample_directory: Path):"""Test recursive directory deletion"""# Add files from directory firstfor f in sample_directory.glob("**/*.txt"):@@ -41,10 +38,10 @@ class TestRm:# Get resource directoryentries = await client.ls("viking://resources/")for data in entries:- if data['isDir']:- dir_uri = data['uri']+ if data["isDir"]:+ dir_uri = data["uri"]await client.rm(dir_uri, recursive=True)- with pytest.raises(Exception):+ with pytest.raises(Exception): # noqa: B017await client.stat(dir_uri)@@ -62,7 +59,7 @@ class TestMv:new_uri = "viking://resources/moved/"await client.mv(uri, new_uri)# Verify original location does not exist- with pytest.raises(Exception):+ with pytest.raises(Exception): # noqa: B017await client.stat(uri)await client.stat(new_uri)@@ -85,11 +82,7 @@ class TestGrep:"""Test case insensitive search"""client, uri = client_with_resource- result = await client.grep(- uri,- pattern="SAMPLE",- case_insensitive=True- )+ result = await client.grep(uri, pattern="SAMPLE", case_insensitive=True)print(result)assert isinstance(result, dict)assert "matches" in result and result["count"] > 0@@ -120,10 +113,7 @@ class TestGlob:client, uri = client_with_resourceparent_uri = "/".join(uri.split("/")[:-1]) + "/"- result = await client.glob(- pattern="*.md",- uri=parent_uri- )+ result = await client.glob(pattern="*.md", uri=parent_uri)assert isinstance(result, dict)assert "matches" in result and result["count"] > 0diff --git a/tests/client/test_filesystem.py b/tests/client/test_filesystem.pyindex e0830311c..ca3572238 100644--- a/tests/client/test_filesystem.py+++ b/tests/client/test_filesystem.py@@ -56,15 +56,15 @@ class TestRead:entries = await client.tree(uri)content = ""for e in entries:- if not e['isDir']:- content = await client.read(e['uri'])+ if not e["isDir"]:+ content = await client.read(e["uri"])assert isinstance(content, str)assert len(content) > 0- assert "Sample Document" in content+ assert "Sample Document" in contentasync def test_read_nonexistent_file(self, client: AsyncOpenViking):"""Test reading nonexistent file"""- with pytest.raises(Exception):+ with pytest.raises(Exception): # noqa: B017await client.read("viking://nonexistent/file.txt")diff --git a/tests/client/test_import_export.py b/tests/client/test_import_export.pyindex 001588917..e4b1b9f13 100644--- a/tests/client/test_import_export.py+++ b/tests/client/test_import_export.py@@ -5,8 +5,6 @@from pathlib import Path-import pytest-from openviking import AsyncOpenViking@@ -33,10 +31,7 @@ class TestExportOvpack:# Export entire resource directoryexport_path = temp_dir / "dir_export.ovpack"- result = await client.export_ovpack(- "viking://resources/",- str(export_path)- )+ result = await client.export_ovpack("viking://resources/", str(export_path))assert isinstance(result, str)@@ -54,9 +49,7 @@ class TestImportOvpack:# Import to new locationimport_uri = await client.import_ovpack(- str(export_path),- "viking://resources/imported/",- vectorize=False+ str(export_path), "viking://resources/imported/", vectorize=False)assert isinstance(import_uri, str)@@ -72,39 +65,30 @@ class TestImportOvpack:# First importawait client.import_ovpack(- str(export_path),- "viking://resources/force_test/",- vectorize=False+ str(export_path), "viking://resources/force_test/", vectorize=False)# Second force import (overwrite)import_uri = await client.import_ovpack(- str(export_path),- "viking://resources/force_test/",- force=True,- vectorize=False+ str(export_path), "viking://resources/force_test/", force=True, vectorize=False)assert isinstance(import_uri, str)-async def test_import_export_roundtrip(self, client: AsyncOpenViking, sample_markdown_file: Path, temp_dir: Path):"""Test export-import roundtrip"""# Add resource- result = await client.add_resource(- path=str(sample_markdown_file),- reason="Roundtrip test"- )+ result = await client.add_resource(path=str(sample_markdown_file), reason="Roundtrip test")original_uri = result["root_uri"]# Read original contentoriginal_content = ""entries = await client.tree(original_uri)for e in entries:- if not e['isDir']:- original_content = await client.read(e['uri'])+ if not e["isDir"]:+ original_content = await client.read(e["uri"])# Exportexport_path = temp_dir / "roundtrip.ovpack"@@ -115,17 +99,15 @@ class TestImportOvpack:# Importimport_uri = await client.import_ovpack(- str(export_path),- "viking://resources/roundtrip/",- vectorize=False+ str(export_path), "viking://resources/roundtrip/", vectorize=False)# Read imported contentimported_content = ""entries = await client.tree(import_uri)for e in entries:- if not e['isDir']:- imported_content = await client.read(e['uri'])+ if not e["isDir"]:+ imported_content = await client.read(e["uri"])# Verify content consistencyassert original_content == imported_contentdiff --git a/tests/client/test_lifecycle.py b/tests/client/test_lifecycle.pyindex 2e1f8d7ee..13a397802 100644--- a/tests/client/test_lifecycle.py+++ b/tests/client/test_lifecycle.py@@ -5,9 +5,6 @@from pathlib import Path-import pytest-import pytest_asyncio-from openviking import AsyncOpenVikingdiff --git a/tests/client/test_relations.py b/tests/client/test_relations.pyindex 9d73e7972..209838c5f 100644--- a/tests/client/test_relations.py+++ b/tests/client/test_relations.py@@ -3,10 +3,6 @@"""Relation tests"""-import pytest--from openviking import AsyncOpenViking-class TestLink:"""Test link creating relations"""@@ -16,11 +12,7 @@ class TestLink:client, uri = client_with_resourcetarget_uri = "viking://resources/target/"- await client.link(- from_uri=uri,- uris=target_uri,- reason="Test link"- )+ await client.link(from_uri=uri, uris=target_uri, reason="Test link")relations = await client.relations(uri)assert any(r.get("uri") == target_uri for r in relations)@@ -28,16 +20,9 @@ class TestLink:async def test_link_multiple_uris(self, client_with_resource):"""Test creating multiple relations"""client, uri = client_with_resource- target_uris = [- "viking://resources/target1/",- "viking://resources/target2/"- ]+ target_uris = ["viking://resources/target1/", "viking://resources/target2/"]- await client.link(- from_uri=uri,- uris=target_uris,- reason="Test multiple links"- )+ await client.link(from_uri=uri, uris=target_uris, reason="Test multiple links")relations = await client.relations(uri)for target in target_uris:@@ -49,11 +34,7 @@ class TestLink:target_uri = "viking://resources/reason_test/"reason = "This is a test reason for the link"- await client.link(- from_uri=uri,- uris=target_uri,- reason=reason- )+ await client.link(from_uri=uri, uris=target_uri, reason=reason)relations = await client.relations(uri)link = next((r for r in relations if r.get("uri") == target_uri), None)@@ -88,10 +69,7 @@ class TestUnlink:client, uri = client_with_resource# Should not raise exception- await client.unlink(- from_uri=uri,- uri="viking://nonexistent/"- )+ await client.unlink(from_uri=uri, uri="viking://nonexistent/")class TestRelations:diff --git a/tests/client/test_resource_management.py b/tests/client/test_resource_management.pyindex 269f23a61..c82fd8a42 100644--- a/tests/client/test_resource_management.py+++ b/tests/client/test_resource_management.py@@ -5,22 +5,15 @@from pathlib import Path-import pytest-from openviking import AsyncOpenVikingclass TestAddResource:"""Test add_resource"""- async def test_add_resource_success(- self, client: AsyncOpenViking, sample_markdown_file: Path- ):+ async def test_add_resource_success(self, client: AsyncOpenViking, sample_markdown_file: Path):"""Test successful resource addition"""- result = await client.add_resource(- path=str(sample_markdown_file),- reason="Test resource"- )+ result = await client.add_resource(path=str(sample_markdown_file), reason="Test resource")assert "root_uri" in resultassert result["root_uri"].startswith("viking://")@@ -44,9 +37,7 @@ class TestAddResource:):"""Test adding resource without waiting (async mode)"""result = await client.add_resource(- path=str(sample_markdown_file),- reason="Test resource",- wait=False+ path=str(sample_markdown_file), reason="Test resource", wait=False)assert "root_uri" in result@@ -61,7 +52,7 @@ class TestAddResource:result = await client.add_resource(path=str(sample_markdown_file),target="viking://resources/custom/",- reason="Test resource"+ reason="Test resource",)assert "root_uri" in result@@ -69,12 +60,9 @@ class TestAddResource:async def test_add_resource_file_not_found(self, client: AsyncOpenViking):"""Test adding nonexistent file"""-- res = await client.add_resource(- path="/nonexistent/file.txt",- reason="Test"- )-++ res = await client.add_resource(path="/nonexistent/file.txt", reason="Test")+assert "errors" in res and len(res["errors"]) > 0@@ -85,10 +73,7 @@ class TestWaitProcessed:self, client: AsyncOpenViking, sample_markdown_file: Path):"""Test waiting for processing to complete"""- await client.add_resource(- path=str(sample_markdown_file),- reason="Test"- )+ await client.add_resource(path=str(sample_markdown_file), reason="Test")status = await client.wait_processed(timeout=60.0)diff --git a/tests/client/test_search.py b/tests/client/test_search.pyindex fa97139dc..ff661a25a 100644--- a/tests/client/test_search.py+++ b/tests/client/test_search.py@@ -3,9 +3,6 @@"""Search tests"""-import pytest--from openviking import AsyncOpenVikingfrom openviking.message import TextPart@@ -23,34 +20,25 @@ class TestFind:assert hasattr(result, "skills")assert hasattr(result, "total")-"""Test limiting result count"""result = await client.find(query="test", limit=5)assert len(result.resources) <= 5"""Test search with target URI"""- result = await client.find(- query="sample",- target_uri=uri- )+ result = await client.find(query="sample", target_uri=uri)assert hasattr(result, "resources")"""Test score threshold filtering"""- result = await client.find(- query="sample document",- score_threshold=0.1- )+ result = await client.find(query="sample document", score_threshold=0.1)# Verify all results have score >= thresholdfor res in result.resources:assert res.score >= 0.1"""Test no matching results"""- result = await client.find(- query="completely_random_nonexistent_query_xyz123"- )+ result = await client.find(query="completely_random_nonexistent_query_xyz123")assert result.total >= 0@@ -71,27 +59,18 @@ class TestSearch:# Add some messages to establish contextsession.add_message("user", [TextPart("I need help with testing")])- result = await client.search(- query="testing help",- session=session- )+ result = await client.search(query="testing help", session=session)assert hasattr(result, "resources")"""Test limiting result count"""- result = await client.search(- query="sample",- limit=3- )+ result = await client.search(query="sample", limit=3)assert len(result.resources) <= 3"""Test complex search with target URI"""parent_uri = "/".join(uri.split("/")[:-1]) + "/"- result = await client.search(- query="sample",- target_uri=parent_uri- )+ result = await client.search(query="sample", target_uri=parent_uri)assert hasattr(result, "resources")diff --git a/tests/client/test_skill_management.py b/tests/client/test_skill_management.pyindex e98f15934..40ce64718 100644--- a/tests/client/test_skill_management.py+++ b/tests/client/test_skill_management.py@@ -4,9 +4,6 @@"""Skill management tests"""from pathlib import Path-import tempfile--import pytestfrom openviking import AsyncOpenViking@@ -18,7 +15,8 @@ class TestAddSkill:"""Test adding skill from file"""# Create skill file in SKILL.md formatskill_file = temp_dir / "test_skill.md"- skill_file.write_text("""---+ skill_file.write_text(+ """---name: test-skilldescription: A test skill for unit testingtags:@@ -38,7 +36,8 @@ Use this skill when you need to test skill functionality.1. Step one: Initialize the skill2. Step two: Execute the skill3. Step three: Verify the result-""")+"""+ )result = await client.add_skill(data=skill_file)@@ -71,14 +70,9 @@ This skill was created from a string."description": "A test MCP tool","inputSchema": {"type": "object",- "properties": {- "query": {- "type": "string",- "description": "The search query"- }- },- "required": ["query"]- }+ "properties": {"query": {"type": "string", "description": "The search query"}},+ "required": ["query"],+ },}result = await client.add_skill(data=mcp_tool)@@ -91,7 +85,8 @@ This skill was created from a string.skill_dir.mkdir()# Create SKILL.md- (skill_dir / "SKILL.md").write_text("""---+ (skill_dir / "SKILL.md").write_text(+ """---name: dir-skilldescription: A skill from directorytags:@@ -102,7 +97,8 @@ tags:## InstructionsThis skill was loaded from a directory.-""")+"""+ )# Create auxiliary file(skill_dir / "reference.md").write_text("# Reference\nAdditional reference content.")@@ -120,7 +116,8 @@ class TestSkillSearch:"""Test searching skills"""# Add skill firstskill_file = temp_dir / "search_skill.md"- skill_file.write_text("""---+ skill_file.write_text(+ """---name: search-test-skilldescription: A skill for testing search functionalitytags:@@ -132,7 +129,8 @@ tags:## InstructionsUse this skill to test search functionality.-""")+"""+ )await client.add_skill(data=skill_file)# Search skillsdiff --git a/tests/conftest.py b/tests/conftest.pyindex 668bb57fc..cff8681cf 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -13,7 +13,6 @@ import pytest_asynciofrom openviking import AsyncOpenViking-# Test data root directoryTEST_ROOT = Path(__file__).parentTEST_TMP_DIR = TEST_ROOT / ".tmp"@@ -56,7 +55,8 @@ def sample_text_file(temp_dir: Path) -> Path:def sample_markdown_file(temp_dir: Path) -> Path:"""Create sample Markdown file"""file_path = temp_dir / "sample.md"- file_path.write_text("""# Sample Document+ file_path.write_text(+ """# Sample Document## IntroductionThis is a sample markdown document for testing OpenViking.@@ -68,7 +68,8 @@ This is a sample markdown document for testing OpenViking.## UsageUse this document to test various OpenViking functionalities.-""")+"""+ )return file_path@@ -76,7 +77,8 @@ Use this document to test various OpenViking functionalities.def sample_skill_file(temp_dir: Path) -> Path:"""Create sample skill file in SKILL.md format"""file_path = temp_dir / "sample_skill.md"- file_path.write_text("""---+ file_path.write_text(+ """---name: sample-skilldescription: A sample skill for testing OpenViking skill managementtags:@@ -96,7 +98,8 @@ Use this skill when you need to test skill functionality.1. Step one: Initialize the skill2. Step two: Execute the skill3. Step three: Verify the result-""")+"""+ )return file_path@@ -122,7 +125,8 @@ def sample_files(temp_dir: Path) -> list[Path]:files = []for i in range(3):file_path = temp_dir / f"batch_file_{i}.md"- file_path.write_text(f"""# Batch File {i}+ file_path.write_text(+ f"""# Batch File {i}## ContentThis is batch file number {i} for testing batch operations.@@ -131,22 +135,21 @@ This is batch file number {i} for testing batch operations.- batch- test- file{i}-""")+"""+ )files.append(file_path)return files# ============ Client Fixtures ============+@pytest_asyncio.fixture(scope="function")async def client(test_data_dir: Path) -> AsyncGenerator[AsyncOpenViking, None]:"""Create initialized OpenViking client"""await AsyncOpenViking.reset()- client = AsyncOpenViking(- path=str(test_data_dir),- user="test_user"- )+ client = AsyncOpenViking(path=str(test_data_dir), user="test_user")await client.initialize()yield client@@ -160,10 +163,7 @@ async def uninitialized_client(test_data_dir: Path) -> AsyncGenerator[AsyncOpenV"""Create uninitialized OpenViking client (for testing initialization flow)"""await AsyncOpenViking.reset()- client = AsyncOpenViking(- path=str(test_data_dir),- user="test_user"- )+ client = AsyncOpenViking(path=str(test_data_dir), user="test_user")yield client@@ -176,14 +176,11 @@ async def uninitialized_client(test_data_dir: Path) -> AsyncGenerator[AsyncOpenV@pytest_asyncio.fixture(scope="function")async def client_with_resource_sync(- client: AsyncOpenViking,- sample_markdown_file: Path+ client: AsyncOpenViking, sample_markdown_file: Path) -> AsyncGenerator[tuple[AsyncOpenViking, str], None]:"""Create client with resource (sync mode, wait for vectorization)"""result = await client.add_resource(- path=str(sample_markdown_file),- reason="Test resource",- wait=True+ path=str(sample_markdown_file), reason="Test resource", wait=True)uri = result.get("root_uri", "")@@ -192,13 +189,9 @@ async def client_with_resource_sync(@pytest_asyncio.fixture(scope="function")async def client_with_resource(- client: AsyncOpenViking,- sample_markdown_file: Path+ client: AsyncOpenViking, sample_markdown_file: Path) -> AsyncGenerator[tuple[AsyncOpenViking, str], None]:"""Create client with resource (async mode, no wait for vectorization)"""- result = await client.add_resource(- path=str(sample_markdown_file),- reason="Test resource"- )+ result = await client.add_resource(path=str(sample_markdown_file), reason="Test resource")uri = result.get("root_uri", "")yield client, uridiff --git a/tests/integration/test_full_workflow.py b/tests/integration/test_full_workflow.pyindex 269e4f937..3e2c1107d 100644--- a/tests/integration/test_full_workflow.py+++ b/tests/integration/test_full_workflow.py@@ -5,7 +5,6 @@from pathlib import Path-import pytestimport pytest_asynciofrom openviking import AsyncOpenViking@@ -17,10 +16,7 @@ async def integration_client(test_data_dir: Path):"""Integration test client"""await AsyncOpenViking.reset()- client = AsyncOpenViking(- path=str(test_data_dir),- user="integration_test_user"- )+ client = AsyncOpenViking(path=str(test_data_dir), user="integration_test_user")await client.initialize()yield client@@ -59,12 +55,7 @@ class TestResourceToSearchWorkflow:client = integration_client# 1. Add resource- add_result = await client.add_resource(- path=str(sample_markdown_file),- reason="Workflow test",- wait=True- )- uri = add_result["root_uri"]+ await client.add_resource(path=str(sample_markdown_file), reason="Workflow test", wait=True)# 2. Searchsearch_result = await client.find(query="sample document")@@ -77,10 +68,10 @@ class TestResourceToSearchWorkflow:else:res = await client.tree(search_result.resources[0].uri)for data in res:- if not data['isDir']:- content = await client.read(data['uri'])+ if not data["isDir"]:+ content = await client.read(data["uri"])assert len(content) > 0-+class TestSessionWorkflow:"""Session management full workflow"""@@ -92,12 +83,9 @@ class TestSessionWorkflow:client = integration_client# 1. Add resource- result = await client.add_resource(- path=str(sample_markdown_file),- reason="Session workflow test",- wait=True+ await client.add_resource(+ path=str(sample_markdown_file), reason="Session workflow test", wait=True)- resource_uri = result["root_uri"]# 2. Create sessionsession = client.session(session_id="workflow_test_session")@@ -166,8 +154,8 @@ class TestImportExportWorkflow:original_content = ""entries = await client.tree(original_uri)for data in entries:- if not data['isDir']:- original_content += await client.read(data['uri'])+ if not data["isDir"]:+ original_content += await client.read(data["uri"])# 3. Exportexport_path = temp_dir / "workflow_export.ovpack"@@ -179,17 +167,15 @@ class TestImportExportWorkflow:# 5. Importimport_uri = await client.import_ovpack(- str(export_path),- "viking://resources/imported/",- vectorize=False+ str(export_path), "viking://resources/imported/", vectorize=False)# 6. Verify content consistencyimported_content = ""entries = await client.tree(import_uri)for data in entries:- if not data['isDir']:- imported_content += await client.read(data['uri'])+ if not data["isDir"]:+ imported_content += await client.read(data["uri"])assert original_content == imported_content@@ -242,8 +228,7 @@ class TestFullEndToEndWorkflow:# Import to new locationimport_uri = await client.import_ovpack(- str(export_path),- "viking://resources/e2e_imported/"+ str(export_path), "viking://resources/e2e_imported/")# Verify import successdiff --git a/tests/misc/test_config_validation.py b/tests/misc/test_config_validation.pyindex 147ae6064..35e50b0b1 100644--- a/tests/misc/test_config_validation.py+++ b/tests/misc/test_config_validation.py@@ -4,11 +4,13 @@"""Test if config validators work correctly"""import sys+from openviking.utils.config.agfs_config import AGFSConfig+from openviking.utils.config.embedding_config import EmbeddingConfig, EmbeddingModelConfigfrom openviking.utils.config.vectordb_config import VectorDBBackendConfig-from openviking.utils.config.embedding_config import EmbeddingModelConfig, EmbeddingConfigfrom openviking.utils.config.vlm_config import VLMConfig+def test_agfs_validation():"""Test AGFS config validation"""print("=" * 60)@@ -27,7 +29,7 @@ def test_agfs_validation():print("\n2. Test invalid backend...")try:config = AGFSConfig(backend="invalid")- print(f" Should fail but passed")+ print(" Should fail but passed")except ValueError as e:print(f" Correctly raised exception: {e}")@@ -35,7 +37,7 @@ def test_agfs_validation():print("\n3. Test S3 backend missing required fields...")try:config = AGFSConfig(backend="s3")- print(f" Should fail but passed")+ print(" Should fail but passed")except ValueError as e:print(f" Correctly raised exception: {e}")@@ -47,12 +49,13 @@ def test_agfs_validation():s3_bucket="my-bucket",s3_region="us-west-1",s3_access_key="fake-access-key-for-testing",- s3_secret_key="fake-secret-key-for-testing-12345"+ s3_secret_key="fake-secret-key-for-testing-12345",)- print(f" Pass")+ print(" Pass")except ValueError as e:print(f" Fail: {e}")+def test_vectordb_validation():"""Test VectorDB config validation"""print("\n" + "=" * 60)@@ -62,34 +65,31 @@ def test_vectordb_validation():# Test 1: local backend missing pathprint("\n1. Test local backend missing path...")try:- config = VectorDBBackendConfig(backend="local", path=None)- print(f" Should fail but passed")+ _ = VectorDBBackendConfig(backend="local", path=None)+ print(" Should fail but passed")except ValueError as e:print(f" Correctly raised exception: {e}")# Test 2: http backend missing urlprint("\n2. Test http backend missing url...")try:- config = VectorDBBackendConfig(backend="http", url=None)- print(f" Should fail but passed")+ _ = VectorDBBackendConfig(backend="http", url=None)+ print(" Should fail but passed")except ValueError as e:print(f" Correctly raised exception: {e}")# Test 3: volcengine backend complete configprint("\n3. Test volcengine backend complete config...")try:- config = VectorDBBackendConfig(+ _ = VectorDBBackendConfig(backend="volcengine",- volcengine={- "ak": "test_ak",- "sk": "test_sk",- "region": "cn-beijing"- }+ volcengine={"ak": "test_ak", "sk": "test_sk", "region": "cn-beijing"},)- print(f" Pass")+ print(" Pass")except ValueError as e:print(f" Fail: {e}")+def test_embedding_validation():"""Test Embedding config validation"""print("\n" + "=" * 60)@@ -99,36 +99,33 @@ def test_embedding_validation():# Test 1: no embedder configprint("\n1. Test no embedder config...")try:- config = EmbeddingConfig()- print(f" Should fail but passed")+ _ = EmbeddingConfig()+ print(" Should fail but passed")except ValueError as e:print(f" Correctly raised exception: {e}")# Test 2: OpenAI provider missing api_keyprint("\n2. Test OpenAI provider missing api_key...")try:- config = EmbeddingConfig(- dense=EmbeddingModelConfig(- provider="openai",- model="text-embedding-3-small"- )+ _ = EmbeddingConfig(+ dense=EmbeddingModelConfig(provider="openai", model="text-embedding-3-small"))- print(f" Should fail but passed")+ print(" Should fail but passed")except ValueError as e:print(f" Correctly raised exception: {e}")# Test 3: OpenAI provider complete configprint("\n3. Test OpenAI provider complete config...")try:- config = EmbeddingConfig(+ _ = EmbeddingConfig(dense=EmbeddingModelConfig(provider="openai",model="text-embedding-3-small",api_key="fake-api-key-for-testing",- dimension=1536+ dimension=1536,))- print(f" Pass")+ print(" Pass")except ValueError as e:print(f" Fail: {e}")@@ -136,29 +133,27 @@ def test_embedding_validation():print("\n4. Test Embedding Provider/Backend sync...")# Case A: Only backend provided -> provider should be syncedconfig_a = EmbeddingModelConfig(- backend="openai",- model="text-embedding-3-small",- api_key="test-key",- dimension=1536+ backend="openai", model="text-embedding-3-small", api_key="test-key", dimension=1536)if config_a.provider == "openai":- print(f" Pass (backend='openai' -> provider='openai')")+ print(" Pass (backend='openai' -> provider='openai')")else:print(f" Fail (backend='openai' -> provider='{config_a.provider}')")# Case B: Both provided -> provider takes precedenceconfig_b = EmbeddingModelConfig(provider="volcengine",- backend="openai", # Conflicting backend+ backend="openai", # Conflicting backendmodel="doubao",api_key="test-key",- dimension=1024+ dimension=1024,)if config_b.provider == "volcengine":- print(f" Pass (provider='volcengine' priority over backend='openai')")+ print(" Pass (provider='volcengine' priority over backend='openai')")else:print(f" Fail (provider='volcengine' should have priority, got '{config_b.provider}')")+def test_vlm_validation():"""Test VLM config validation"""print("\n" + "=" * 60)@@ -168,56 +163,46 @@ def test_vlm_validation():# Test 1: VLM not configured (optional)print("\n1. Test VLM not configured (optional)...")try:- config = VLMConfig()- print(f" Pass (VLM is optional)")+ _ = VLMConfig()+ print(" Pass (VLM is optional)")except ValueError as e:print(f" Fail: {e}")# Test 2: VLM partial config (has model but no api_key)print("\n2. Test VLM partial config...")try:- config = VLMConfig(model="gpt-4")- print(f" Should fail but passed")+ _ = VLMConfig(model="gpt-4")+ print(" Should fail but passed")except ValueError as e:print(f" Correctly raised exception: {e}")# Test 3: VLM complete configprint("\n3. Test VLM complete config...")try:- config = VLMConfig(- model="gpt-4",- api_key="fake-api-key-for-testing",- provider="openai"- )- print(f" Pass")+ _ = VLMConfig(model="gpt-4", api_key="fake-api-key-for-testing", provider="openai")+ print(" Pass")except ValueError as e:print(f" Fail: {e}")# Test 4: VLM Provider/Backend syncprint("\n4. Test VLM Provider/Backend sync...")# Case A: Only backend provided -> provider should be synced- config_a = VLMConfig(- backend="openai",- model="gpt-4",- api_key="test-key"- )+ config_a = VLMConfig(backend="openai", model="gpt-4", api_key="test-key")if config_a.provider == "openai":- print(f" Pass (backend='openai' -> provider='openai')")+ print(" Pass (backend='openai' -> provider='openai')")else:print(f" Fail (backend='openai' -> provider='{config_a.provider}')")# Case B: Both provided -> provider takes precedenceconfig_b = VLMConfig(- provider="volcengine",- backend="openai",- model="doubao",- api_key="test-key"+ provider="volcengine", backend="openai", model="doubao", api_key="test-key")if config_b.provider == "volcengine":- print(f" Pass (provider='volcengine' priority over backend='openai')")+ print(" Pass (provider='volcengine' priority over backend='openai')")else:print(f" Fail (provider='volcengine' should have priority, got '{config_b.provider}')")+if __name__ == "__main__":print("\nStarting config validator tests...\n")@@ -234,5 +219,6 @@ if __name__ == "__main__":except Exception as e:print(f"\nUnexpected error during tests: {e}")import traceback+traceback.print_exc()sys.exit(1)diff --git a/tests/misc/test_vikingdb_observer.py b/tests/misc/test_vikingdb_observer.pyindex 156b88024..0cfbfa168 100644--- a/tests/misc/test_vikingdb_observer.py+++ b/tests/misc/test_vikingdb_observer.py@@ -6,6 +6,7 @@ Test VikingDBObserver functionality"""import asyncio+import openviking as ovdiff --git a/tests/session/conftest.py b/tests/session/conftest.pyindex 0b1b61f7a..1395738f2 100644--- a/tests/session/conftest.py+++ b/tests/session/conftest.py@@ -5,12 +5,11 @@from typing import AsyncGenerator-import pytestimport pytest_asynciofrom openviking import AsyncOpenViking-from openviking.session import Sessionfrom openviking.message import TextPart, ToolPart+from openviking.session import Session@pytest_asyncio.fixture(scope="function")@@ -42,7 +41,7 @@ async def session_with_messages(client: AsyncOpenViking) -> AsyncGenerator[Sessi@pytest_asyncio.fixture(scope="function")async def session_with_tool_call(- client: AsyncOpenViking+ client: AsyncOpenViking,) -> AsyncGenerator[tuple[Session, str, str], None]:"""Create Session with tool call"""session = client.session(session_id="test_session_with_tool")@@ -54,12 +53,9 @@ async def session_with_tool_call(tool_uri=f"viking://session/{session.session_id}/tools/{tool_id}",skill_uri="viking://agent/skills/test_skill",tool_input={"param": "value"},- tool_status="running"+ tool_status="running",)- msg = session.add_message("assistant", [- TextPart("Executing tool..."),- tool_part- ])+ msg = session.add_message("assistant", [TextPart("Executing tool..."), tool_part])yield session, msg.id, tool_iddiff --git a/tests/session/test_session_commit.py b/tests/session/test_session_commit.pyindex 1dcf230bb..0f6a90d9a 100644--- a/tests/session/test_session_commit.py+++ b/tests/session/test_session_commit.py@@ -3,11 +3,9 @@"""Commit tests"""-import pytest-from openviking import AsyncOpenViking-from openviking.session import Sessionfrom openviking.message import TextPart+from openviking.session import Sessionclass TestCommit:@@ -21,7 +19,9 @@ class TestCommit:assert result.get("status") == "committed"assert "session_id" in result- async def test_commit_extracts_memories(self, session_with_messages: Session, client: AsyncOpenViking):+ async def test_commit_extracts_memories(+ self, session_with_messages: Session, client: AsyncOpenViking+ ):"""Test commit extracts memories"""result = session_with_messages.commit()diff --git a/tests/session/test_session_context.py b/tests/session/test_session_context.pyindex ecac52182..ceccf3698 100644--- a/tests/session/test_session_context.py+++ b/tests/session/test_session_context.py@@ -3,11 +3,9 @@"""Context retrieval tests"""-import pytest-from openviking import AsyncOpenViking-from openviking.session import Sessionfrom openviking.message import TextPart+from openviking.session import Sessionclass TestGetContextForSearch:@@ -15,19 +13,14 @@ class TestGetContextForSearch:async def test_get_context_basic(self, session_with_messages: Session):"""Test basic context retrieval"""- context = session_with_messages.get_context_for_search(- query="testing help"- )+ context = session_with_messages.get_context_for_search(query="testing help")assert isinstance(context, dict)assert "summaries" in context or "recent_messages" in contextasync def test_get_context_with_max_messages(self, session_with_messages: Session):"""Test limiting max messages"""- context = session_with_messages.get_context_for_search(- query="test",- max_messages=2- )+ context = session_with_messages.get_context_for_search(query="test", max_messages=2)assert isinstance(context, dict)if "recent_messages" in context:@@ -45,10 +38,7 @@ class TestGetContextForSearch:# Add more messagessession.add_message("user", [TextPart("Second message")])- context = session.get_context_for_search(- query="test",- max_archives=1- )+ context = session.get_context_for_search(query="test", max_archives=1)assert isinstance(context, dict)diff --git a/tests/session/test_session_lifecycle.py b/tests/session/test_session_lifecycle.pyindex 4b749dd89..a48c5d127 100644--- a/tests/session/test_session_lifecycle.py+++ b/tests/session/test_session_lifecycle.py@@ -3,8 +3,6 @@"""Session lifecycle tests"""-import pytest-from openviking import AsyncOpenVikingfrom openviking.session import Session@@ -46,7 +44,9 @@ class TestSessionCreate:class TestSessionLoad:"""Test Session loading"""- async def test_load_existing_session(self, session_with_messages: Session, client: AsyncOpenViking):+ async def test_load_existing_session(+ self, session_with_messages: Session, client: AsyncOpenViking+ ):"""Test loading existing session"""session_id = session_with_messages.session_iddiff --git a/tests/session/test_session_messages.py b/tests/session/test_session_messages.pyindex 8fc9587cc..25f2267c7 100644--- a/tests/session/test_session_messages.py+++ b/tests/session/test_session_messages.py@@ -3,10 +3,8 @@"""Message management tests"""-import pytest-+from openviking.message import ContextPart, TextPart, ToolPartfrom openviking.session import Session-from openviking.message import TextPart, ContextPart, ToolPartclass TestAddMessage:@@ -31,10 +29,7 @@ class TestAddMessage:async def test_add_message_with_multiple_parts(self, session: Session):"""Test adding message with multiple parts"""- parts = [- TextPart("Here is some context:"),- TextPart("And here is more text.")- ]+ parts = [TextPart("Here is some context:"), TextPart("And here is more text.")]msg = session.add_message("assistant", parts)assert len(msg.parts) == 2@@ -46,8 +41,8 @@ class TestAddMessage:ContextPart(uri="viking://user/test/resources/doc.md",context_type="resource",- abstract="Some context abstract"- )+ abstract="Some context abstract",+ ),]msg = session.add_message("assistant", parts)@@ -61,12 +56,9 @@ class TestAddMessage:tool_uri="viking://session/test/tools/tool_123",skill_uri="viking://agent/skills/search",tool_input={"query": "test"},- tool_status="running"+ tool_status="running",)- msg = session.add_message("assistant", [- TextPart("Executing search..."),- tool_part- ])+ msg = session.add_message("assistant", [TextPart("Executing search..."), tool_part])assert len(msg.parts) == 2@@ -91,7 +83,7 @@ class TestUpdateToolPart:message_id=message_id,tool_id=tool_id,output="Tool execution completed successfully",- status="completed"+ status="completed",)# Verify tool status updated@@ -107,7 +99,7 @@ class TestUpdateToolPart:message_id=message_id,tool_id=tool_id,output="Tool execution failed: error message",- status="failed"+ status="failed",)# Verify tool status updateddiff --git a/tests/session/test_session_usage.py b/tests/session/test_session_usage.pyindex 480043c75..d9cc5a11c 100644--- a/tests/session/test_session_usage.py+++ b/tests/session/test_session_usage.py@@ -3,10 +3,8 @@"""Usage record tests"""-import pytest--from openviking.session import Sessionfrom openviking.message import TextPart+from openviking.session import Sessionclass TestUsed:@@ -18,10 +16,12 @@ class TestUsed:session.add_message("user", [TextPart("Test message")])# Record used contexts- session.used(contexts=[- "viking://user/test/resources/doc1.md",- "viking://user/test/resources/doc2.md"- ])+ session.used(+ contexts=[+ "viking://user/test/resources/doc1.md",+ "viking://user/test/resources/doc2.md",+ ]+ )# Verify usage recordsassert len(session.usage_records) > 0@@ -30,10 +30,7 @@ class TestUsed:"""Test recording used skill"""session.add_message("user", [TextPart("Test message")])- session.used(skill={- "uri": "viking://agent/skills/search",- "name": "search_skill"- })+ session.used(skill={"uri": "viking://agent/skills/search", "name": "search_skill"})assert len(session.usage_records) > 0@@ -43,10 +40,7 @@ class TestUsed:session.used(contexts=["viking://user/test/resources/doc.md"],- skill={- "uri": "viking://agent/skills/analyze",- "name": "analyze_skill"- }+ skill={"uri": "viking://agent/skills/analyze", "name": "analyze_skill"},)assert len(session.usage_records) > 0diff --git a/tests/vectordb/benchmark_stress.py b/tests/vectordb/benchmark_stress.pyindex 1168a557c..016618c11 100644--- a/tests/vectordb/benchmark_stress.py+++ b/tests/vectordb/benchmark_stress.py@@ -1,16 +1,14 @@import argparse+import osimport random-import time-import threadingimport shutil-import os-import sys+import threading+import time+import numpy as np-from typing import List, Dict, Any-from concurrent.futures import ThreadPoolExecutor-from openviking.storage.vectordb.collection.local_collection import get_or_create_local_collectionfrom openviking.storage.vectordb.collection.collection import Collection+from openviking.storage.vectordb.collection.local_collection import get_or_create_local_collection# --- Configuration ---DEFAULT_DIM = 128@@ -18,6 +16,7 @@ DEFAULT_DB_PATH = "./benchmark_stress_db"CATEGORIES = ["news", "sports", "finance", "tech", "entertainment"]TAGS = ["hot", "new", "archived", "premium", "public"]+class StressStats:def __init__(self):self.lock = threading.Lock()@@ -48,75 +47,91 @@ class StressStats:with self.lock:duration = time.time() - self.start_timeprint(f"\n--- Stress Test Report (Duration: {duration:.2f}s) ---")- print(f"Insert: {self.insert_count} ops, {self.insert_count/duration:.2f} OPS, Avg Latency: {np.mean(self.insert_latency) if self.insert_latency else 0:.4f}s")- print(f"Search: {self.search_count} ops, {self.search_count/duration:.2f} OPS, Avg Latency: {np.mean(self.search_latency) if self.search_latency else 0:.4f}s")- print(f"Delete: {self.delete_count} ops, {self.delete_count/duration:.2f} OPS, Avg Latency: {np.mean(self.delete_latency) if self.delete_latency else 0:.4f}s")+ print(+ f"Insert: {self.insert_count} ops, {self.insert_count / duration:.2f} OPS, Avg Latency: {np.mean(self.insert_latency) if self.insert_latency else 0:.4f}s"+ )+ print(+ f"Search: {self.search_count} ops, {self.search_count / duration:.2f} OPS, Avg Latency: {np.mean(self.search_latency) if self.search_latency else 0:.4f}s"+ )+ print(+ f"Delete: {self.delete_count} ops, {self.delete_count / duration:.2f} OPS, Avg Latency: {np.mean(self.delete_latency) if self.delete_latency else 0:.4f}s"+ )print(f"Total Ops: {self.insert_count + self.search_count + self.delete_count}")print("------------------------------------------------")+def generate_random_vector(dim):return [random.random() for _ in range(dim)]+def generate_random_sparse_vector():# Random sparse vector: few random terms with weights- terms = ["term" + str(i) for i in range(100)] # Vocabulary of 100 terms+ terms = ["term" + str(i) for i in range(100)] # Vocabulary of 100 termsnum_terms = random.randint(1, 10)selected = random.sample(terms, num_terms)return {term: random.random() for term in selected}+def setup_collection(path: str, dim: int, enable_sparse: bool):if os.path.exists(path):shutil.rmtree(path)-+fields = [{"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},{"FieldName": "vector", "FieldType": "vector", "Dim": dim},{"FieldName": "category", "FieldType": "string"},{"FieldName": "score", "FieldType": "float32"},{"FieldName": "is_active", "FieldType": "bool"},- {"FieldName": "tags", "FieldType": "list<string>"}+ {"FieldName": "tags", "FieldType": "list<string>"},]-+if enable_sparse:fields.append({"FieldName": "sparse_vec", "FieldType": "sparse_vector"})meta_data = {"CollectionName": "stress_collection","Description": "A collection for violent stress testing",- "Fields": fields+ "Fields": fields,}-+col = get_or_create_local_collection(meta_data=meta_data, path=path)-+# Create multiple indexes to stress the index managervector_index_config = {- "IndexType": "flat_hybrid" if enable_sparse else "flat",+ "IndexType": "flat_hybrid" if enable_sparse else "flat","Distance": "l2",# "FieldName": "vector" # FieldName is not in Pydantic schema for VectorIndexConfig}if enable_sparse:vector_index_config["EnableSparse"] = True- col.create_index("idx_vector", {- "IndexName": "idx_vector",- "VectorIndex": vector_index_config- })-+ col.create_index("idx_vector", {"IndexName": "idx_vector", "VectorIndex": vector_index_config})+# col.create_index("idx_category", {# "IndexName": "idx_category",- # "ScalarIndex": ["category"]+ # "ScalarIndex": ["category"]# })-+return col-def worker_insert(col: Collection, stats: StressStats, start_id: int, count: int, batch_size: int, dim: int, enable_sparse: bool, stop_event: threading.Event = None):++def worker_insert(+ col: Collection,+ stats: StressStats,+ start_id: int,+ count: int,+ batch_size: int,+ dim: int,+ enable_sparse: bool,+ stop_event: threading.Event = None,+):current_id = start_idend_id = start_id + count-+while current_id < end_id:if stop_event and stop_event.is_set():break-+batch_data = []real_batch_size = min(batch_size, end_id - current_id)for i in range(real_batch_size):@@ -126,65 +141,91 @@ def worker_insert(col: Collection, stats: StressStats, start_id: int, count: int"category": random.choice(CATEGORIES),"score": random.random(),"is_active": random.choice([True, False]),- "tags": random.sample(TAGS, k=random.randint(1, 3))+ "tags": random.sample(TAGS, k=random.randint(1, 3)),}if enable_sparse:item["sparse_vec"] = generate_random_sparse_vector()batch_data.append(item)-+t0 = time.time()try:col.upsert_data(batch_data)stats.record_insert(time.time() - t0)except Exception as e:print(f"[Insert] Error: {e}")-+current_id += real_batch_size# time.sleep(0.01) # Small sleep to prevent total lock starvation if any-def worker_search(col: Collection, stats: StressStats, duration: int, dim: int, max_id: int, enable_sparse: bool, stop_event: threading.Event = None):++def worker_search(+ col: Collection,+ stats: StressStats,+ duration: int,+ dim: int,+ max_id: int,+ enable_sparse: bool,+ stop_event: threading.Event = None,+):end_time = time.time() + durationwhile time.time() < end_time:if stop_event and stop_event.is_set():break-+t0 = time.time()try:# 1. Pure Vector Searchif enable_sparse:sparse_q = generate_random_sparse_vector()- col.search_by_vector("idx_vector", generate_random_vector(dim), limit=10, sparse_vector=sparse_q)+ col.search_by_vector(+ "idx_vector", generate_random_vector(dim), limit=10, sparse_vector=sparse_q+ )else:col.search_by_vector("idx_vector", generate_random_vector(dim), limit=10)-+# 2. Filtered Vector Search (Complex)filter_cond = {"category": {"in": [random.choice(CATEGORIES)]},"score": {"gt": 0.5},- "is_active": {"eq": True}+ "is_active": {"eq": True},}if enable_sparse:sparse_q = generate_random_sparse_vector()- col.search_by_vector("idx_vector", generate_random_vector(dim), limit=10, filters=filter_cond, sparse_vector=sparse_q)+ col.search_by_vector(+ "idx_vector",+ generate_random_vector(dim),+ limit=10,+ filters=filter_cond,+ sparse_vector=sparse_q,+ )else:- col.search_by_vector("idx_vector", generate_random_vector(dim), limit=10, filters=filter_cond)-+ col.search_by_vector(+ "idx_vector", generate_random_vector(dim), limit=10, filters=filter_cond+ )+# 3. Fetch Datarandom_ids = [random.randint(0, max_id) for _ in range(5)]col.fetch_data(random_ids)-+stats.record_search(time.time() - t0)except Exception as e:print(f"[Search] Error: {e}")-+# time.sleep(0.005)-def worker_delete(col: Collection, stats: StressStats, duration: int, max_id: int, stop_event: threading.Event = None):++def worker_delete(+ col: Collection,+ stats: StressStats,+ duration: int,+ max_id: int,+ stop_event: threading.Event = None,+):end_time = time.time() + durationwhile time.time() < end_time:if stop_event and stop_event.is_set():break-+t0 = time.time()try:# Randomly delete a small batch@@ -193,8 +234,9 @@ def worker_delete(col: Collection, stats: StressStats, duration: int, max_id: instats.record_delete(time.time() - t0)except Exception as e:print(f"[Delete] Error: {e}")-- time.sleep(0.1) # Delete less frequently than insert/search++ time.sleep(0.1) # Delete less frequently than insert/search+def run_stress_test():parser = argparse.ArgumentParser(description="Violent Vectordb Stress Test")@@ -206,58 +248,93 @@ def run_stress_test():parser.add_argument("--search_threads", type=int, default=8, help="Number of search threads")parser.add_argument("--delete_threads", type=int, default=2, help="Number of delete threads")parser.add_argument("--enable_sparse", action="store_true", help="Enable sparse vector support")-+args = parser.parse_args()-- print(f"=== Starting Stress Test (Dim={args.dim}, Duration={args.duration}s, Sparse={args.enable_sparse}) ===")++ print(+ f"=== Starting Stress Test (Dim={args.dim}, Duration={args.duration}s, Sparse={args.enable_sparse}) ==="+ )print(f"DB Path: {args.path}")-+col = setup_collection(args.path, args.dim, args.enable_sparse)stats = StressStats()-+# Preload dataprint(f"Preloading {args.initial_data} items...")- worker_insert(col, stats, 0, args.initial_data, batch_size=100, dim=args.dim, enable_sparse=args.enable_sparse)+ worker_insert(+ col,+ stats,+ 0,+ args.initial_data,+ batch_size=100,+ dim=args.dim,+ enable_sparse=args.enable_sparse,+ )print("Preload complete.")-+# Reset stats for the actual stress phase- stats = StressStats()-+ stats = StressStats()+# Define ID ranges for inserts to avoid massive collisions, though collisions are also fun for testing# Let's make inserts append new datastart_id_base = args.initial_data- items_per_thread = 1000000 # Large enough to keep running-+ items_per_thread = 1000000 # Large enough to keep running+threads = []-+# Stop Eventstop_event = threading.Event()-+# Start Insert Threadsfor i in range(args.insert_threads):t_start = start_id_base + i * items_per_thread- t = threading.Thread(target=worker_insert, args=(col, stats, t_start, items_per_thread, 50, args.dim, args.enable_sparse, stop_event))+ t = threading.Thread(+ target=worker_insert,+ args=(+ col,+ stats,+ t_start,+ items_per_thread,+ 50,+ args.dim,+ args.enable_sparse,+ stop_event,+ ),+ )t.daemon = Truet.start()threads.append(t)-+# Start Search Threads- # They will query random IDs up to current approximate max.+ # They will query random IDs up to current approximate max.# We estimate max_id conservatively to avoid too many misses, but misses are okay.estimated_max_id = start_id_base + (items_per_thread * args.insert_threads)- for i in range(args.search_threads):- t = threading.Thread(target=worker_search, args=(col, stats, args.duration, args.dim, estimated_max_id, args.enable_sparse, stop_event))+ for _ in range(args.search_threads):+ t = threading.Thread(+ target=worker_search,+ args=(+ col,+ stats,+ args.duration,+ args.dim,+ estimated_max_id,+ args.enable_sparse,+ stop_event,+ ),+ )t.daemon = Truet.start()threads.append(t)-+# Start Delete Threads- for i in range(args.delete_threads):- t = threading.Thread(target=worker_delete, args=(col, stats, args.duration, estimated_max_id, stop_event))+ for _ in range(args.delete_threads):+ t = threading.Thread(+ target=worker_delete, args=(col, stats, args.duration, estimated_max_id, stop_event)+ )t.daemon = Truet.start()threads.append(t)-+# Monitor loopstart_time = time.time()try:@@ -267,17 +344,17 @@ def run_stress_test():breakexcept KeyboardInterrupt:print("\nInterrupted by user")-+print("\nStopping threads...")stop_event.set()-+# Wait for all threads to finishfor t in threads:t.join(timeout=2.0)-+# Threads are daemon, will die when main exits, but let's give a quick report firststats.report()-+# Cleanupprint("Cleaning up...")col.close()@@ -285,5 +362,6 @@ def run_stress_test():shutil.rmtree(args.path)print("Done.")+if __name__ == "__main__":run_stress_test()diff --git a/tests/vectordb/test_bytes_row.py b/tests/vectordb/test_bytes_row.pyindex 1bd8b60cc..31b32b16f 100644--- a/tests/vectordb/test_bytes_row.py+++ b/tests/vectordb/test_bytes_row.py@@ -3,6 +3,7 @@import unittestfrom dataclasses import dataclass, fieldfrom typing import List+from openviking.storage.vectordb.store.serializable import serializable@@ -16,12 +17,7 @@ class TestBytesRow(unittest.TestCase):active: bool = Falsename: str = ""- data = BasicData(- id=1234567890,- score=0.95,- active=True,- name="viking_db"- )+ data = BasicData(id=1234567890, score=0.95, active=True, name="viking_db")# Serializeserialized = data.serialize()@@ -50,9 +46,7 @@ class TestBytesRow(unittest.TestCase):counts: List[int] = field(default_factory=list)data = ListData(- tags=["AI", "Vector", "Search"],- embedding=[0.1, 0.2, 0.3, 0.4],- counts=[1, 10, 100]+ tags=["AI", "Vector", "Search"], embedding=[0.1, 0.2, 0.3, 0.4], counts=[1, 10, 100])serialized = data.serialize()@@ -97,11 +91,12 @@ class TestBytesRow(unittest.TestCase):class BinaryData:raw: bytes = b""- blob = b"\x00\x01\x02\xFF\xFE"+ blob = b"\x00\x01\x02\xff\xfe"data = BinaryData(raw=blob)serialized = data.serialize()val = BinaryData.bytes_row.deserialize_field(serialized, "raw")self.assertEqual(val, blob)+if __name__ == "__main__":unittest.main()diff --git a/tests/vectordb/test_collection_large_scale.py b/tests/vectordb/test_collection_large_scale.pyindex 2ac8af9e5..85cac5c3e 100644--- a/tests/vectordb/test_collection_large_scale.py+++ b/tests/vectordb/test_collection_large_scale.py@@ -5,13 +5,11 @@ Large scale scenario tests - Collection stress tests and performance testsTests for large data volumes, high-dimensional vectors, complex queries, etc."""-import unittest-import shutil-import os+import gcimport random+import shutilimport time-import gc-from typing import List, Dict, Any+import unittestfrom openviking.storage.vectordb.collection.local_collection import get_or_create_local_collection@@ -68,7 +66,6 @@ class TestLargeScaleScenarios(unittest.TestCase):# Prepare 10K dataprint("Preparing 10,000 records...")random.seed(42)- data_list = []batch_size = 1000total_records = 10000@@ -79,17 +76,21 @@ class TestLargeScaleScenarios(unittest.TestCase):batch_data = []for i in range(batch_size):record_id = batch_idx * batch_size + i + 1- batch_data.append({- "id": record_id,- "vector": [random.gauss(0, 1) for _ in range(128)],- "category": f"cat_{record_id % 20}",- "score": random.uniform(0, 100),- "timestamp": int(time.time()) + record_id,- })+ batch_data.append(+ {+ "id": record_id,+ "vector": [random.gauss(0, 1) for _ in range(128)],+ "category": f"cat_{record_id % 20}",+ "score": random.uniform(0, 100),+ "timestamp": int(time.time()) + record_id,+ }+ )result = collection.upsert_data(batch_data)self.assertEqual(len(result.ids), batch_size)- print(f" Batch {batch_idx + 1}/{total_records // batch_size} inserted ({len(result.ids)} records)")+ print(+ f" Batch {batch_idx + 1}/{total_records // batch_size} inserted ({len(result.ids)} records)"+ )insert_time = time.time() - start_timeprint(f"β Inserted {total_records} records in {insert_time:.2f}s")@@ -104,7 +105,7 @@ class TestLargeScaleScenarios(unittest.TestCase):"IndexName": "idx_10k","VectorIndex": {"IndexType": "flat", "Distance": "ip"},"ScalarIndex": ["category", "score", "timestamp"],- }+ },)index_time = time.time() - index_startprint(f"β Index created in {index_time:.2f}s")@@ -114,7 +115,7 @@ class TestLargeScaleScenarios(unittest.TestCase):query_vec = [random.gauss(0, 1) for _ in range(128)]search_times = []- for i in range(10):+ for _ in range(10):search_start = time.time()result = collection.search_by_vector("idx_10k", dense_vector=query_vec, limit=100)search_time = time.time() - search_start@@ -122,7 +123,7 @@ class TestLargeScaleScenarios(unittest.TestCase):self.assertEqual(len(result.data), 100)avg_search_time = sum(search_times) / len(search_times)- print(f"β Average search time: {avg_search_time*1000:.2f}ms (10 queries, top-100)")+ print(f"β Average search time: {avg_search_time * 1000:.2f}ms (10 queries, top-100)")# Test filtered searchprint("Testing filtered search...")@@ -131,10 +132,12 @@ class TestLargeScaleScenarios(unittest.TestCase):"idx_10k",dense_vector=query_vec,limit=50,- filters={"op": "range", "field": "score", "gte": 50.0}+ filters={"op": "range", "field": "score", "gte": 50.0},)filter_time = time.time() - filter_start- print(f"β Filtered search completed in {filter_time*1000:.2f}ms (returned {len(result.data)} results)")+ print(+ f"β Filtered search completed in {filter_time * 1000:.2f}ms (returned {len(result.data)} results)"+ )def test_02_large_batch_insert_50k(self):"""Test extra large batch insert - 50,000 records"""@@ -164,11 +167,13 @@ class TestLargeScaleScenarios(unittest.TestCase):batch_data = []for i in range(batch_size):record_id = batch_idx * batch_size + i + 1- batch_data.append({- "id": record_id,- "vector": [random.uniform(-1, 1) for _ in range(256)],- "group": f"group_{record_id % 50}",- })+ batch_data.append(+ {+ "id": record_id,+ "vector": [random.uniform(-1, 1) for _ in range(256)],+ "group": f"group_{record_id % 50}",+ }+ )collection.upsert_data(batch_data)if (batch_idx + 1) % 5 == 0:@@ -187,7 +192,7 @@ class TestLargeScaleScenarios(unittest.TestCase):"IndexName": "idx_50k","VectorIndex": {"IndexType": "flat", "Distance": "l2"},"ScalarIndex": ["group"],- }+ },)index_time = time.time() - index_startprint(f"β Index created in {index_time:.2f}s")@@ -230,11 +235,13 @@ class TestLargeScaleScenarios(unittest.TestCase):record_id = batch_idx * batch_size + i + 1# Generate high-dimensional vectorvector = [random.gauss(0, 1) for _ in range(1024)]- batch_data.append({- "id": record_id,- "vector": vector,- "label": f"label_{record_id % 10}",- })+ batch_data.append(+ {+ "id": record_id,+ "vector": vector,+ "label": f"label_{record_id % 10}",+ }+ )collection.upsert_data(batch_data)if (batch_idx + 1) % 4 == 0:@@ -250,7 +257,7 @@ class TestLargeScaleScenarios(unittest.TestCase):{"IndexName": "idx_high_dim","VectorIndex": {"IndexType": "flat", "Distance": "cosine"},- }+ },)query_vec = [random.gauss(0, 1) for _ in range(1024)]@@ -259,7 +266,7 @@ class TestLargeScaleScenarios(unittest.TestCase):search_time = time.time() - search_startself.assertEqual(len(result.data), 50)- print(f"β High-dim search completed in {search_time*1000:.2f}ms (top-50)")+ print(f"β High-dim search completed in {search_time * 1000:.2f}ms (top-50)")def test_04_massive_updates(self):"""Test massive updates - 10,000 records batch update"""@@ -295,7 +302,7 @@ class TestLargeScaleScenarios(unittest.TestCase):insert_start = time.time()batch_size = 1000for i in range(0, len(data_list), batch_size):- collection.upsert_data(data_list[i:i+batch_size])+ collection.upsert_data(data_list[i : i + batch_size])insert_time = time.time() - insert_startprint(f"β Initial insert completed in {insert_time:.2f}s")@@ -306,7 +313,7 @@ class TestLargeScaleScenarios(unittest.TestCase):"IndexName": "idx_update","VectorIndex": {"IndexType": "flat"},"ScalarIndex": ["version"],- }+ },)# Batch update@@ -322,7 +329,7 @@ class TestLargeScaleScenarios(unittest.TestCase):update_start = time.time()for i in range(0, len(update_data), batch_size):- collection.upsert_data(update_data[i:i+batch_size])+ collection.upsert_data(update_data[i : i + batch_size])if (i + batch_size) % 2000 == 0:print(f" Updated {i + batch_size} / {total_records} records")update_time = time.time() - update_start@@ -379,7 +386,7 @@ class TestLargeScaleScenarios(unittest.TestCase):delete_start = time.time()for i in range(0, len(delete_ids), delete_batch_size):- batch_ids = delete_ids[i:i+delete_batch_size]+ batch_ids = delete_ids[i : i + delete_batch_size]collection.delete_data(batch_ids)if (i + delete_batch_size) % 5000 == 0:print(f" Deleted {i + delete_batch_size} / 25000 records")@@ -391,7 +398,7 @@ class TestLargeScaleScenarios(unittest.TestCase):# Verify deletion - Method 1: fetch_datafetch_result = collection.fetch_data([1, 100, 25000, 25001, 50000])self.assertEqual(len(fetch_result.items), 2) # Only 25001 and 50000 exist- self.assertEqual(set([item.id for item in fetch_result.items]), {25001, 50000})+ self.assertEqual({item.id for item in fetch_result.items}, {25001, 50000})print("β Delete verification passed (fetch_data)")# Verify deletion - Method 2: verify actual retrievable record count through search@@ -399,20 +406,24 @@ class TestLargeScaleScenarios(unittest.TestCase):search_result = collection.search_by_vector("idx",dense_vector=[random.random() for _ in range(32)],- limit=30000 # Request more than remaining count+ limit=30000, # Request more than remaining count)actual_count = len(search_result.data)print(f" Search returned {actual_count} records (expected ~25000)")# Index may have delayed updates, so we allow some margin# But should at least be less than original count- self.assertLess(actual_count, 30000, "Search should return less than 30000 records after deletion")+ self.assertLess(+ actual_count, 30000, "Search should return less than 30000 records after deletion"+ )# Verify deleted records are not in search resultssearch_ids = {item.id for item in search_result.data}deleted_samples = [1, 100, 1000, 10000, 25000]for deleted_id in deleted_samples:- self.assertNotIn(deleted_id, search_ids, f"Deleted ID {deleted_id} should not be in search results")+ self.assertNotIn(+ deleted_id, search_ids, f"Deleted ID {deleted_id} should not be in search results"+ )print(f"β Deletion verified through search: {actual_count} records remain")# Verify deletion - Method 3: aggregate_data to count remaining data@@ -420,7 +431,11 @@ class TestLargeScaleScenarios(unittest.TestCase):agg_result = collection.aggregate_data("idx", op="count")agg_count = agg_result.agg.get("_total", 0)print(f" Aggregate count: {agg_count}")- self.assertEqual(agg_count, 25000, f"Expected 25000 records after deletion, but aggregate_data returned {agg_count}")+ self.assertEqual(+ agg_count,+ 25000,+ f"Expected 25000 records after deletion, but aggregate_data returned {agg_count}",+ )print("β Aggregate count verification passed")def test_06_complex_multi_filter_large_scale(self):@@ -456,14 +471,16 @@ class TestLargeScaleScenarios(unittest.TestCase):batch_data = []for i in range(batch_size):record_id = batch_idx * batch_size + i + 1- batch_data.append({- "id": record_id,- "vector": [random.gauss(0, 1) for _ in range(128)],- "category": random.choice(categories),- "priority": random.randint(1, 10),- "score": random.uniform(0, 100),- "status": random.choice(statuses),- })+ batch_data.append(+ {+ "id": record_id,+ "vector": [random.gauss(0, 1) for _ in range(128)],+ "category": random.choice(categories),+ "priority": random.randint(1, 10),+ "score": random.uniform(0, 100),+ "status": random.choice(statuses),+ }+ )collection.upsert_data(batch_data)print(f"β Inserted {total_records} records")@@ -475,7 +492,7 @@ class TestLargeScaleScenarios(unittest.TestCase):"IndexName": "idx_complex","VectorIndex": {"IndexType": "flat", "Distance": "ip"},"ScalarIndex": ["category", "priority", "score", "status"],- }+ },)# Test complex filter queries@@ -489,14 +506,18 @@ class TestLargeScaleScenarios(unittest.TestCase):{"op": "range", "field": "priority", "gte": 7},{"op": "range", "field": "score", "gt": 50.0},{"op": "must", "field": "status", "conds": ["active"]},- ]+ ],}query_vec = [random.gauss(0, 1) for _ in range(128)]search_start = time.time()- result1 = collection.search_by_vector("idx_complex", dense_vector=query_vec, limit=100, filters=filter1)+ result1 = collection.search_by_vector(+ "idx_complex", dense_vector=query_vec, limit=100, filters=filter1+ )search_time1 = time.time() - search_start- print(f" Query 1 (4 conditions): {len(result1.data)} results in {search_time1*1000:.2f}ms")+ print(+ f" Query 1 (4 conditions): {len(result1.data)} results in {search_time1 * 1000:.2f}ms"+ )# Query 2: (category="C" OR category="D") AND priority IN [3,5,7] AND status != "inactive"filter2 = {@@ -505,13 +526,17 @@ class TestLargeScaleScenarios(unittest.TestCase):{"op": "must", "field": "category", "conds": ["C", "D"]},{"op": "must", "field": "priority", "conds": [3, 5, 7]},{"op": "must_not", "field": "status", "conds": ["inactive"]},- ]+ ],}search_start = time.time()- result2 = collection.search_by_vector("idx_complex", dense_vector=query_vec, limit=100, filters=filter2)+ result2 = collection.search_by_vector(+ "idx_complex", dense_vector=query_vec, limit=100, filters=filter2+ )search_time2 = time.time() - search_start- print(f" Query 2 (3 conditions): {len(result2.data)} results in {search_time2*1000:.2f}ms")+ print(+ f" Query 2 (3 conditions): {len(result2.data)} results in {search_time2 * 1000:.2f}ms"+ )# Query 3: Range query + sortsearch_start = time.time()@@ -520,10 +545,12 @@ class TestLargeScaleScenarios(unittest.TestCase):field="score",order="desc",limit=500,- filters={"op": "range", "field": "priority", "gte": 5, "lte": 8}+ filters={"op": "range", "field": "priority", "gte": 5, "lte": 8},)search_time3 = time.time() - search_start- print(f" Query 3 (scalar sort): {len(result3.data)} results in {search_time3*1000:.2f}ms")+ print(+ f" Query 3 (scalar sort): {len(result3.data)} results in {search_time3 * 1000:.2f}ms"+ )print("β Complex multi-filter queries completed")@@ -560,13 +587,15 @@ class TestLargeScaleScenarios(unittest.TestCase):batch_data = []for i in range(batch_size):record_id = batch_idx * batch_size + i + 1- batch_data.append({- "id": record_id,- "vector": [random.random() for _ in range(64)],- "country": random.choice(countries),- "city": random.choice(cities),- "product": random.choice(products),- })+ batch_data.append(+ {+ "id": record_id,+ "vector": [random.random() for _ in range(64)],+ "country": random.choice(countries),+ "city": random.choice(cities),+ "product": random.choice(products),+ }+ )collection.upsert_data(batch_data)print(f"β Inserted {total_records} records")@@ -578,7 +607,7 @@ class TestLargeScaleScenarios(unittest.TestCase):"IndexName": "idx_agg","VectorIndex": {"IndexType": "flat"},"ScalarIndex": ["country", "city", "product"],- }+ },)# Test aggregations with different cardinalities@@ -588,19 +617,25 @@ class TestLargeScaleScenarios(unittest.TestCase):agg_start = time.time()result1 = collection.aggregate_data("idx_agg", op="count", field="country")agg_time1 = time.time() - agg_start- print(f" Country aggregation (50 groups): {len(result1.agg)} groups in {agg_time1*1000:.2f}ms")+ print(+ f" Country aggregation (50 groups): {len(result1.agg)} groups in {agg_time1 * 1000:.2f}ms"+ )# Medium cardinality aggregation (100 groups)agg_start = time.time()result2 = collection.aggregate_data("idx_agg", op="count", field="product")agg_time2 = time.time() - agg_start- print(f" Product aggregation (100 groups): {len(result2.agg)} groups in {agg_time2*1000:.2f}ms")+ print(+ f" Product aggregation (100 groups): {len(result2.agg)} groups in {agg_time2 * 1000:.2f}ms"+ )# High cardinality aggregation (200 groups)agg_start = time.time()result3 = collection.aggregate_data("idx_agg", op="count", field="city")agg_time3 = time.time() - agg_start- print(f" City aggregation (200 groups): {len(result3.agg)} groups in {agg_time3*1000:.2f}ms")+ print(+ f" City aggregation (200 groups): {len(result3.agg)} groups in {agg_time3 * 1000:.2f}ms"+ )# Filtered aggregationagg_start = time.time()@@ -608,10 +643,14 @@ class TestLargeScaleScenarios(unittest.TestCase):"idx_agg",op="count",field="product",- filters={"op": "must", "field": "country", "conds": ["country_0", "country_1", "country_2"]}+ filters={+ "op": "must",+ "field": "country",+ "conds": ["country_0", "country_1", "country_2"],+ },)agg_time4 = time.time() - agg_start- print(f" Filtered aggregation: {len(result4.agg)} groups in {agg_time4*1000:.2f}ms")+ print(f" Filtered aggregation: {len(result4.agg)} groups in {agg_time4 * 1000:.2f}ms")print("β Large scale aggregation tests completed")@@ -641,11 +680,13 @@ class TestLargeScaleScenarios(unittest.TestCase):batch_data = []for i in range(batch_size):record_id = batch_idx * batch_size + i + 1- batch_data.append({- "id": record_id,- "vector": [random.uniform(-1, 1) for _ in range(256)],- "metadata": f"metadata_{record_id}",- })+ batch_data.append(+ {+ "id": record_id,+ "vector": [random.uniform(-1, 1) for _ in range(256)],+ "metadata": f"metadata_{record_id}",+ }+ )collection1.upsert_data(batch_data)write_time = time.time() - write_start@@ -659,12 +700,14 @@ class TestLargeScaleScenarios(unittest.TestCase):"IndexName": "idx_persist","VectorIndex": {"IndexType": "flat", "Distance": "l2"},"ScalarIndex": ["metadata"],- }+ },)# Execute search and record resultsquery_vec = [random.uniform(-1, 1) for _ in range(256)]- result_before = collection1.search_by_vector("idx_persist", dense_vector=query_vec, limit=50)+ result_before = collection1.search_by_vector(+ "idx_persist", dense_vector=query_vec, limit=50+ )ids_before = [item.id for item in result_before.data]# Close@@ -677,9 +720,7 @@ class TestLargeScaleScenarios(unittest.TestCase):# Phase 2: Reloadprint("\nPhase 2: Reloading from disk...")reload_start = time.time()- collection2 = self.register_collection(- get_or_create_local_collection(path=TEST_DB_PATH)- )+ collection2 = self.register_collection(get_or_create_local_collection(path=TEST_DB_PATH))reload_time = time.time() - reload_startprint(f"β Collection reloaded in {reload_time:.2f}s")@@ -702,7 +743,6 @@ class TestLargeScaleScenarios(unittest.TestCase):self.assertEqual(agg_result.agg["_total"], total_records)print(f"β Total count verified: {agg_result.agg['_total']}")-def test_09_concurrent_operations_simulation(self):"""Simulate concurrent operations scenario (serial simulation)"""print("\n=== Test 9: Concurrent Operations Simulation ===")@@ -739,7 +779,7 @@ class TestLargeScaleScenarios(unittest.TestCase):"IndexName": "idx_concurrent","VectorIndex": {"IndexType": "flat"},"ScalarIndex": ["category"],- }+ },)print("β Initial setup completed (5000 records)")@@ -748,30 +788,40 @@ class TestLargeScaleScenarios(unittest.TestCase):operations = []# 100 searches- for i in range(100):+ for _ in range(100):query_vec = [random.random() for _ in range(64)]operations.append(("search", query_vec))# 50 insertsfor i in range(50):new_id = 5001 + i- operations.append(("insert", {- "id": new_id,- "vector": [random.random() for _ in range(64)],- "category": f"cat_{new_id % 10}",- }))+ operations.append(+ (+ "insert",+ {+ "id": new_id,+ "vector": [random.random() for _ in range(64)],+ "category": f"cat_{new_id % 10}",+ },+ )+ )# 30 updates- for i in range(30):+ for _ in range(30):update_id = random.randint(1, 5000)- operations.append(("update", {- "id": update_id,- "vector": [random.random() for _ in range(64)],- "category": f"cat_updated",- }))+ operations.append(+ (+ "update",+ {+ "id": update_id,+ "vector": [random.random() for _ in range(64)],+ "category": "cat_updated",+ },+ )+ )# 20 deletes- for i in range(20):+ for _ in range(20):delete_id = random.randint(1, 1000)operations.append(("delete", delete_id))@@ -786,7 +836,7 @@ class TestLargeScaleScenarios(unittest.TestCase):for op_type, op_data in operations:if op_type == "search":- result = collection.search_by_vector("idx_concurrent", dense_vector=op_data, limit=10)+ collection.search_by_vector("idx_concurrent", dense_vector=op_data, limit=10)search_count += 1elif op_type == "insert":collection.upsert_data([op_data])@@ -837,7 +887,9 @@ class TestLargeScaleScenarios(unittest.TestCase):]collection.upsert_data(batch_data)- collection.create_index("idx_stress", {"IndexName": "idx_stress", "VectorIndex": {"IndexType": "flat"}})+ collection.create_index(+ "idx_stress", {"IndexName": "idx_stress", "VectorIndex": {"IndexType": "flat"}}+ )print("β Initial setup completed")# Stress test: run for 5 seconds@@ -849,7 +901,9 @@ class TestLargeScaleScenarios(unittest.TestCase):while time.time() - start_time < test_duration:# Randomly select operation type- op = random.choice(["search", "search", "search", "insert", "update"]) # Higher weight for search+ op = random.choice(+ ["search", "search", "search", "insert", "update"]+ ) # Higher weight for searchif op == "search":query_vec = [random.gauss(0, 1) for _ in range(128)]@@ -859,16 +913,24 @@ class TestLargeScaleScenarios(unittest.TestCase):search_times.append(search_time)elif op == "insert":new_id = random.randint(10001, 20000)- collection.upsert_data([{- "id": new_id,- "vector": [random.gauss(0, 1) for _ in range(128)],- }])+ collection.upsert_data(+ [+ {+ "id": new_id,+ "vector": [random.gauss(0, 1) for _ in range(128)],+ }+ ]+ )elif op == "update":update_id = random.randint(1, 10000)- collection.upsert_data([{- "id": update_id,- "vector": [random.gauss(0, 1) for _ in range(128)],- }])+ collection.upsert_data(+ [+ {+ "id": update_id,+ "vector": [random.gauss(0, 1) for _ in range(128)],+ }+ ]+ )operation_count += 1@@ -881,7 +943,9 @@ class TestLargeScaleScenarios(unittest.TestCase):avg_search = sum(search_times) / len(search_times)min_search = min(search_times)max_search = max(search_times)- print(f" Search latency: avg={avg_search*1000:.2f}ms, min={min_search*1000:.2f}ms, max={max_search*1000:.2f}ms")+ print(+ f" Search latency: avg={avg_search * 1000:.2f}ms, min={min_search * 1000:.2f}ms, max={max_search * 1000:.2f}ms"+ )def run_large_scale_tests():@@ -890,21 +954,21 @@ def run_large_scale_tests():runner = unittest.TextTestRunner(verbosity=2)result = runner.run(suite)- print("\n" + "="*70)+ print("\n" + "=" * 70)print("Large Scale Test Summary:")print(f"Total tests run: {result.testsRun}")print(f"Successes: {result.testsRun - len(result.failures) - len(result.errors)}")print(f"Failures: {len(result.failures)}")print(f"Errors: {len(result.errors)}")- print("="*70)+ print("=" * 70)return result.wasSuccessful()if __name__ == "__main__":- print("\n" + "="*70)+ print("\n" + "=" * 70)print("OpenViking Collection - Large Scale Scenario Tests")- print("="*70)+ print("=" * 70)success = run_large_scale_tests()exit(0 if success else 1)diff --git a/tests/vectordb/test_crash_recovery.py b/tests/vectordb/test_crash_recovery.pyindex 940055ea8..88585cad8 100644--- a/tests/vectordb/test_crash_recovery.py+++ b/tests/vectordb/test_crash_recovery.py@@ -3,17 +3,16 @@import multiprocessingimport osimport shutil+import sysimport timeimport unittest-import sys-import random-from typing import List, Dict, Anyfrom openviking.storage.vectordb.collection.local_collection import get_or_create_local_collectionDB_PATH_CRASH = "./test_db_crash_recovery"DB_PATH_ROBUST = "./test_db_robust_crash"+def worker_write_and_crash(path, start_id, count, event_ready):"""Subprocess function:@@ -29,35 +28,37 @@ def worker_write_and_crash(path, start_id, count, event_ready):"Fields": [{"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},{"FieldName": "vector", "FieldType": "vector", "Dim": 4},- {"FieldName": "data", "FieldType": "string"}- ]+ {"FieldName": "data", "FieldType": "string"},+ ],}# Force a fresh instance in this processcol = get_or_create_local_collection(meta_data=meta_data, path=path)-+if not col.has_index("idx_crash"):- col.create_index("idx_crash", {- "IndexName": "idx_crash",- "VectorIndex": {"IndexType": "flat", "Distance": "l2"}- })-+ col.create_index(+ "idx_crash",+ {"IndexName": "idx_crash", "VectorIndex": {"IndexType": "flat", "Distance": "l2"}},+ )+# Write datadata = []for i in range(count):uid = start_id + i- data.append({- "id": uid,- "vector": [0.1] * 4, # Use constant vector for easier search verification- "data": f"crash_data_{uid}"- })-+ data.append(+ {+ "id": uid,+ "vector": [0.1] * 4, # Use constant vector for easier search verification+ "data": f"crash_data_{uid}",+ }+ )+print(f"[Subprocess] Upserting {count} items...")col.upsert_data(data)- print(f"[Subprocess] Upsert done. Not closing.")-+ print("[Subprocess] Upsert done. Not closing.")+# Notify main process that write is doneevent_ready.set()-+# Simulate work or wait to be killed# DO NOT close collection# We sleep long enough for parent to kill us@@ -66,6 +67,7 @@ def worker_write_and_crash(path, start_id, count, event_ready):print(f"[Subprocess] Error: {e}")sys.exit(1)+def setup_robust_collection(path):"""Helper to setup collection config for robust test"""meta_data = {@@ -74,17 +76,18 @@ def setup_robust_collection(path):{"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},{"FieldName": "vector", "FieldType": "vector", "Dim": 4},{"FieldName": "data", "FieldType": "string"},- {"FieldName": "tag", "FieldType": "string"}- ]+ {"FieldName": "tag", "FieldType": "string"},+ ],}col = get_or_create_local_collection(meta_data=meta_data, path=path)if not col.has_index("idx_main"):- col.create_index("idx_main", {- "IndexName": "idx_main",- "VectorIndex": {"IndexType": "flat", "Distance": "l2"}- })+ col.create_index(+ "idx_main",+ {"IndexName": "idx_main", "VectorIndex": {"IndexType": "flat", "Distance": "l2"}},+ )return col+def worker_cycle_1_write(path, event_ready):"""Cycle 1: Write 500 items (ID 0-499).@@ -92,26 +95,22 @@ def worker_cycle_1_write(path, event_ready):"""try:col = setup_robust_collection(path)-+data = []for i in range(500):- data.append({- "id": i,- "vector": [0.1] * 4,- "data": f"cycle1_{i}",- "tag": "c1"- })-+ data.append({"id": i, "vector": [0.1] * 4, "data": f"cycle1_{i}", "tag": "c1"})+print("[Cycle 1] Upserting 500 items...")col.upsert_data(data)print("[Cycle 1] Upsert done.")-+event_ready.set()- time.sleep(60) # Wait to be killed+ time.sleep(60) # Wait to be killedexcept Exception as e:print(f"[Cycle 1] Error: {e}")sys.exit(1)+def worker_cycle_2_mix(path, event_ready):"""Cycle 2:@@ -122,37 +121,33 @@ def worker_cycle_2_mix(path, event_ready):"""try:col = setup_robust_collection(path)-+# Verify Cycle 1 data exists before modificationres = col.fetch_data([0, 499])if len(res.items) != 2:print("[Cycle 2] Critical: Cycle 1 data missing on startup!")sys.exit(1)-+# Delete 0-99print("[Cycle 2] Deleting 100 items (0-99)...")col.delete_data(list(range(100)))-+# Upsert 1000-1299print("[Cycle 2] Upserting 300 items (1000-1299)...")data = []for i in range(300):uid = 1000 + i- data.append({- "id": uid,- "vector": [0.2] * 4,- "data": f"cycle2_{uid}",- "tag": "c2"- })+ data.append({"id": uid, "vector": [0.2] * 4, "data": f"cycle2_{uid}", "tag": "c2"})col.upsert_data(data)print("[Cycle 2] Operations done.")-+event_ready.set()- time.sleep(60) # Wait to be killed+ time.sleep(60) # Wait to be killedexcept Exception as e:print(f"[Cycle 2] Error: {e}")sys.exit(1)+class TestCrashRecovery(unittest.TestCase):def setUp(self):if os.path.exists(DB_PATH_CRASH):@@ -172,82 +167,82 @@ class TestCrashRecovery(unittest.TestCase):(where close() was not called)."""print("\n=== Test Simple Crash Recovery ===")-+# Use 'spawn' to ensure clean process state (especially important for RocksDB/LevelDB locks)- ctx = multiprocessing.get_context('spawn')+ ctx = multiprocessing.get_context("spawn")event = ctx.Event()-+data_count = 100p = ctx.Process(target=worker_write_and_crash, args=(DB_PATH_CRASH, 0, data_count, event))p.start()-+# Wait for write to complete in subprocessprint("[Main] Waiting for subprocess to write data...")is_set = event.wait(timeout=5)self.assertTrue(is_set, "Subprocess timed out writing data")-+# Give it a tiny moment to ensure the OS flush might happen (though we want to test robustness)# But immediate kill is what we want to test.# However, Python's LevelDB binding might be sync or async. Usually writes go to OS cache.- time.sleep(0.5)-+ time.sleep(0.5)+# KILL the process immediately to simulate crash (no cleanup, no flush)print("[Main] Terminating subprocess (Simulating Crash)...")p.terminate()p.join()print(f"[Main] Subprocess terminated with exit code: {p.exitcode}")-+# Recoverprint("[Main] Recovering collection in main process...")# Re-opening the collection should trigger recovery logic (PersistentIndex should rebuild/catchup from Store)col = get_or_create_local_collection(path=DB_PATH_CRASH)-+# 1. Verify Data Persistence (Store)print("[Main] Verifying data fetch (All 100 items)...")# Check all IDsall_ids = list(range(data_count))res = col.fetch_data(all_ids)-+print(f"[Main] Fetch retrieved {len(res.items)} items. Missing: {len(res.ids_not_exist)}")-- self.assertEqual(len(res.items), data_count, f"Should find all {data_count} items in KV Store")-++ self.assertEqual(+ len(res.items), data_count, f"Should find all {data_count} items in KV Store"+ )+print("[Main] Data fetch verified.")-+# 2. Verify Index Recovery- # Since we killed the process, the Index (which might be in memory or partially written)+ # Since we killed the process, the Index (which might be in memory or partially written)# needs to be reconstructed from the KV Store delta during initialization.print("[Main] Verifying vector search (Index Recovery)...")-+# Searching for vector [0.1, 0.1, 0.1, 0.1] should return our data- search_res = col.search_by_vector(- "idx_crash",- dense_vector=[0.1]*4,- limit=data_count- )-+ search_res = col.search_by_vector("idx_crash", dense_vector=[0.1] * 4, limit=data_count)+found_ids = [item.id for item in search_res.data]print(f"[Main] Search returned {len(found_ids)} items.")-+# We expect all items to be found if index recovery works- self.assertEqual(len(found_ids), data_count, "Index should contain all items after recovery")-+ self.assertEqual(+ len(found_ids), data_count, "Index should contain all items after recovery"+ )+col.close()print("[Main] Simple recovery test passed.")def run_process_and_crash(self, target_func):- ctx = multiprocessing.get_context('spawn')+ ctx = multiprocessing.get_context("spawn")event = ctx.Event()p = ctx.Process(target=target_func, args=(DB_PATH_ROBUST, event))p.start()-+# Wait for work doneis_set = event.wait(timeout=5)self.assertTrue(is_set, "Subprocess timed out")-+# Give a split second for OS buffers (simulate sudden power loss/crash)time.sleep(0.5)-+print(f"[Main] Crashing process {p.pid}...")p.terminate()p.join()@@ -255,71 +250,72 @@ class TestCrashRecovery(unittest.TestCase):def test_multi_cycle_crash(self):print("\n=== Test Robust Multi-Cycle Crash Recovery ===")-+# --- Cycle 1: Write & Crash ---print("\n--- Starting Cycle 1 ---")self.run_process_and_crash(worker_cycle_1_write)-+# Verify Cycle 1 Recoveryprint("[Main] Verifying Cycle 1 recovery...")col = setup_robust_collection(DB_PATH_ROBUST)-+# Check counts- res_search = col.search_by_vector("idx_main", [0.1]*4, limit=1000)+ res_search = col.search_by_vector("idx_main", [0.1] * 4, limit=1000)self.assertEqual(len(res_search.data), 500, "Should have 500 items after Cycle 1")col.close()-+# --- Cycle 2: Mix Ops & Crash ---print("\n--- Starting Cycle 2 ---")self.run_process_and_crash(worker_cycle_2_mix)-+# Verify Cycle 2 Recoveryprint("[Main] Verifying Cycle 2 recovery...")col = setup_robust_collection(DB_PATH_ROBUST)-+# 1. Verify Deletions (0-99 should be gone)print("[Main] Verifying deletions (0-99)...")res_deleted = col.fetch_data(list(range(10)))self.assertEqual(len(res_deleted.items), 0, "IDs 0-9 should be deleted")-+# Search check for deleted items- # IDs 0-99 had vector [0.1]*4.+ # IDs 0-99 had vector [0.1]*4.# IDs 100-499 still have [0.1]*4.# So searching [0.1]*4 should return 400 items that match perfectly (score ~0)# But vector search returns all items up to limit.- res_search_old = col.search_by_vector("idx_main", [0.1]*4, limit=1000)-+ res_search_old = col.search_by_vector("idx_main", [0.1] * 4, limit=1000)+# Verify total count is 700 (400 old + 300 new)self.assertEqual(len(res_search_old.data), 700, "Total index items should be 700")# Verify IDs 0-99 are GONE- found_ids = set(item.id for item in res_search_old.data)+ found_ids = {item.id for item in res_search_old.data}for i in range(100):self.assertNotIn(i, found_ids, f"ID {i} should have been deleted")-+# Verify IDs 100-499 EXISTfor i in range(100, 500):- self.assertIn(i, found_ids, f"ID {i} should exist")+ self.assertIn(i, found_ids, f"ID {i} should exist")# 2. Verify New Inserts (1000-1299 should exist)print("[Main] Verifying new inserts (1000-1299)...")res_new = col.fetch_data([1000, 1299])self.assertEqual(len(res_new.items), 2, "New items 1000 and 1299 should exist")-+# Search check for new items ([0.2]*4)# Should also return all 700 items, but sorted differently- res_search_new = col.search_by_vector("idx_main", [0.2]*4, limit=1000)+ res_search_new = col.search_by_vector("idx_main", [0.2] * 4, limit=1000)self.assertEqual(len(res_search_new.data), 700, "Should return all 700 items")-+# Verify IDs 1000-1299 EXIST in the result- found_ids_new = set(item.id for item in res_search_new.data)+ found_ids_new = {item.id for item in res_search_new.data}for i in range(1000, 1300):self.assertIn(i, found_ids_new, f"ID {i} should exist")-- print(f"[Main] Total items verified via index: 700")-++ print("[Main] Total items verified via index: 700")+col.close()print("\n[Main] Robust recovery test passed.")+if __name__ == "__main__":unittest.main()diff --git a/tests/vectordb/test_filter_ops.py b/tests/vectordb/test_filter_ops.pyindex 617bde4b0..b57f682e3 100644--- a/tests/vectordb/test_filter_ops.py+++ b/tests/vectordb/test_filter_ops.py@@ -1,12 +1,12 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0-import unittest+import gc+import randomimport shutilimport time-import random-import string-import gc+import unittest+from openviking.storage.vectordb.collection.local_collection import get_or_create_local_collectiondb_path_basic = "./db_test_filters_basic/"@@ -14,11 +14,14 @@ db_path_complex = "./db_test_filters_complex/"db_path_lifecycle = "./db_test_filters_lifecycle/"db_path_scale = "./db_test_filters_scale/"+def clean_dir(path):shutil.rmtree(path, ignore_errors=True)+class TestFilterOpsBasic(unittest.TestCase):"""Basic Filter operator tests"""+def setUp(self):clean_dir(db_path_basic)self.path = db_path_basic@@ -46,11 +49,41 @@ class TestFilterOpsBasic(unittest.TestCase):def _insert_data(self):data = [- {"id": 1, "embedding": [1.0, 0, 0, 0], "val_int": 10, "val_float": 1.1, "val_str": "apple"},- {"id": 2, "embedding": [1.0, 0, 0, 0], "val_int": 20, "val_float": 2.2, "val_str": "banana"},- {"id": 3, "embedding": [1.0, 0, 0, 0], "val_int": 30, "val_float": 3.3, "val_str": "cherry"},- {"id": 4, "embedding": [1.0, 0, 0, 0], "val_int": 40, "val_float": 4.4, "val_str": "date"},- {"id": 5, "embedding": [1.0, 0, 0, 0], "val_int": 50, "val_float": 5.5, "val_str": "elderberry"},+ {+ "id": 1,+ "embedding": [1.0, 0, 0, 0],+ "val_int": 10,+ "val_float": 1.1,+ "val_str": "apple",+ },+ {+ "id": 2,+ "embedding": [1.0, 0, 0, 0],+ "val_int": 20,+ "val_float": 2.2,+ "val_str": "banana",+ },+ {+ "id": 3,+ "embedding": [1.0, 0, 0, 0],+ "val_int": 30,+ "val_float": 3.3,+ "val_str": "cherry",+ },+ {+ "id": 4,+ "embedding": [1.0, 0, 0, 0],+ "val_int": 40,+ "val_float": 4.4,+ "val_str": "date",+ },+ {+ "id": 5,+ "embedding": [1.0, 0, 0, 0],+ "val_int": 50,+ "val_float": 5.5,+ "val_str": "elderberry",+ },]self.collection.upsert_data(data)@@ -64,10 +97,7 @@ class TestFilterOpsBasic(unittest.TestCase):def _search(self, filters):res = self.collection.search_by_vector(- "idx_basic",- dense_vector=[1.0, 0, 0, 0],- limit=100,- filters=filters+ "idx_basic", dense_vector=[1.0, 0, 0, 0], limit=100, filters=filters)return sorted([item.id for item in res.data])@@ -79,11 +109,14 @@ class TestFilterOpsBasic(unittest.TestCase):# Prefixself.assertEqual(self._search({"op": "prefix", "field": "val_str", "prefix": "ap"}), [1])# Contains- self.assertEqual(self._search({"op": "contains", "field": "val_str", "substring": "er"}), [3, 5]) # chERry, eldERbERry+ self.assertEqual(+ self._search({"op": "contains", "field": "val_str", "substring": "er"}), [3, 5]+ ) # chERry, eldERbERryclass TestFilterOpsComplex(unittest.TestCase):"""Complex mixed logic tests"""+def setUp(self):clean_dir(db_path_complex)self.path = db_path_complex@@ -105,7 +138,10 @@ class TestFilterOpsComplex(unittest.TestCase):{"FieldName": "category", "FieldType": "string"},{"FieldName": "tags", "FieldType": "string"},{"FieldName": "price", "FieldType": "int64"},- {"FieldName": "rating_int", "FieldType": "int64"}, # Use int64 instead of float32 to avoid potential bug+ {+ "FieldName": "rating_int",+ "FieldType": "int64",+ }, # Use int64 instead of float32 to avoid potential bug],}return get_or_create_local_collection(meta_data=collection_meta, path=self.path)@@ -114,16 +150,86 @@ class TestFilterOpsComplex(unittest.TestCase):# Build some slightly complex data scenarios# id 1-10data = [- {"id": 1, "embedding": [1.0, 0, 0, 0], "category": "electronics", "tags": "mobile,apple,new", "price": 8000, "rating_int": 48},- {"id": 2, "embedding": [1.0, 0, 0, 0], "category": "electronics", "tags": "mobile,android,sale", "price": 3000, "rating_int": 45},- {"id": 3, "embedding": [1.0, 0, 0, 0], "category": "electronics", "tags": "laptop,apple,pro", "price": 15000, "rating_int": 49},- {"id": 4, "embedding": [1.0, 0, 0, 0], "category": "electronics", "tags": "laptop,windows,budget", "price": 4000, "rating_int": 40},- {"id": 5, "embedding": [1.0, 0, 0, 0], "category": "home", "tags": "furniture,sofa", "price": 2000, "rating_int": 42},- {"id": 6, "embedding": [1.0, 0, 0, 0], "category": "home", "tags": "kitchen,blender", "price": 300, "rating_int": 38},- {"id": 7, "embedding": [1.0, 0, 0, 0], "category": "books", "tags": "fiction,sci-fi", "price": 50, "rating_int": 47},- {"id": 8, "embedding": [1.0, 0, 0, 0], "category": "books", "tags": "fiction,fantasy", "price": 60, "rating_int": 46},- {"id": 9, "embedding": [1.0, 0, 0, 0], "category": "clothing", "tags": "shirt,summer", "price": 100, "rating_int": 41},- {"id": 10, "embedding": [1.0, 0, 0, 0], "category": "clothing", "tags": "pants,winter", "price": 200, "rating_int": 43},+ {+ "id": 1,+ "embedding": [1.0, 0, 0, 0],+ "category": "electronics",+ "tags": "mobile,apple,new",+ "price": 8000,+ "rating_int": 48,+ },+ {+ "id": 2,+ "embedding": [1.0, 0, 0, 0],+ "category": "electronics",+ "tags": "mobile,android,sale",+ "price": 3000,+ "rating_int": 45,+ },+ {+ "id": 3,+ "embedding": [1.0, 0, 0, 0],+ "category": "electronics",+ "tags": "laptop,apple,pro",+ "price": 15000,+ "rating_int": 49,+ },+ {+ "id": 4,+ "embedding": [1.0, 0, 0, 0],+ "category": "electronics",+ "tags": "laptop,windows,budget",+ "price": 4000,+ "rating_int": 40,+ },+ {+ "id": 5,+ "embedding": [1.0, 0, 0, 0],+ "category": "home",+ "tags": "furniture,sofa",+ "price": 2000,+ "rating_int": 42,+ },+ {+ "id": 6,+ "embedding": [1.0, 0, 0, 0],+ "category": "home",+ "tags": "kitchen,blender",+ "price": 300,+ "rating_int": 38,+ },+ {+ "id": 7,+ "embedding": [1.0, 0, 0, 0],+ "category": "books",+ "tags": "fiction,sci-fi",+ "price": 50,+ "rating_int": 47,+ },+ {+ "id": 8,+ "embedding": [1.0, 0, 0, 0],+ "category": "books",+ "tags": "fiction,fantasy",+ "price": 60,+ "rating_int": 46,+ },+ {+ "id": 9,+ "embedding": [1.0, 0, 0, 0],+ "category": "clothing",+ "tags": "shirt,summer",+ "price": 100,+ "rating_int": 41,+ },+ {+ "id": 10,+ "embedding": [1.0, 0, 0, 0],+ "category": "clothing",+ "tags": "pants,winter",+ "price": 200,+ "rating_int": 43,+ },]self.collection.upsert_data(data)@@ -137,10 +243,7 @@ class TestFilterOpsComplex(unittest.TestCase):def _search(self, filters):res = self.collection.search_by_vector(- "idx_complex",- dense_vector=[1.0, 0, 0, 0],- limit=100,- filters=filters+ "idx_complex", dense_vector=[1.0, 0, 0, 0], limit=100, filters=filters)return sorted([item.id for item in res.data])@@ -160,10 +263,10 @@ class TestFilterOpsComplex(unittest.TestCase):"op": "or","conds": [{"op": "contains", "field": "tags", "substring": "apple"},- {"op": "range", "field": "rating_int", "gt": 48}- ]- }- ]+ {"op": "range", "field": "rating_int", "gt": 48},+ ],+ },+ ],}self.assertEqual(self._search(filters), [1, 3])@@ -181,8 +284,8 @@ class TestFilterOpsComplex(unittest.TestCase):"conds": [{"op": "range", "field": "price", "lt": 5000},{"op": "must_not", "field": "category", "conds": ["clothing"]},- {"op": "range", "field": "rating_int", "gte": 40}- ]+ {"op": "range", "field": "rating_int", "gte": 40},+ ],}# Price < 5000: 2, 4, 5, 6, 7, 8, 9, 10# Not clothing: 1, 2, 3, 4, 5, 6, 7, 8 (excludes 9, 10)@@ -200,8 +303,8 @@ class TestFilterOpsComplex(unittest.TestCase):"op": "and","conds": [{"op": "prefix", "field": "tags", "prefix": "f"},- {"op": "range", "field": "price", "gte": 50, "lte": 100}- ]+ {"op": "range", "field": "price", "gte": 50, "lte": 100},+ ],}self.assertEqual(self._search(filters), [7, 8])@@ -210,21 +313,21 @@ class TestFilterOpsComplex(unittest.TestCase):# Logic:# ((Category="electronics" OR Category="home") AND# (Price < 1000 OR (Tags contains "fiction" AND Rating > 4.5)))-+# A: Category="electronics" -> 1, 2, 3, 4# B: Category="home" -> 5, 6# A OR B -> 1, 2, 3, 4, 5, 6-+# C: Price < 1000 -> 6(300), 7(50), 8(60), 9(100), 10(200)# D: Tags contains "fiction" -> 7, 8# E: Rating > 4.5 -> 1(48), 3(49), 7(47), 8(46)# D AND E -> 7, 8# C OR (D AND E) -> 6, 7, 8, 9, 10 (7,8 already in C, so union is same as C)-+# Intersection (A OR B) AND (C OR (D AND E)):# {1, 2, 3, 4, 5, 6} INTERSECT {6, 7, 8, 9, 10}# Result: 6-+filters = {"op": "and","conds": [@@ -232,8 +335,8 @@ class TestFilterOpsComplex(unittest.TestCase):"op": "or","conds": [{"op": "must", "field": "category", "conds": ["electronics"]},- {"op": "must", "field": "category", "conds": ["home"]}- ]+ {"op": "must", "field": "category", "conds": ["home"]},+ ],},{"op": "or",@@ -243,12 +346,12 @@ class TestFilterOpsComplex(unittest.TestCase):"op": "and","conds": [{"op": "contains", "field": "tags", "substring": "fiction"},- {"op": "range", "field": "rating_int", "gt": 45}- ]- }- ]- }- ]+ {"op": "range", "field": "rating_int", "gt": 45},+ ],+ },+ ],+ },+ ],}self.assertEqual(self._search(filters), [6])@@ -260,33 +363,25 @@ class TestFilterOpsComplex(unittest.TestCase):# < 3000: 5, 6, 7, 8, 9, 10# Result: 3, 5, 6, 7, 8, 9, 10# (Assuming range_out(gte=3000, lte=8000) means NOT [3000, 8000])-- filters = {- "op": "range_out",- "field": "price",- "gte": 3000,- "lte": 8000- }++ filters = {"op": "range_out", "field": "price", "gte": 3000, "lte": 8000}res = self._search(filters)self.assertEqual(res, [3, 5, 6, 7, 8, 9, 10])-+# range_out combined with Must# (price < 3000 OR price > 8000) AND Category="electronics"# Electronics: 1, 2, 3, 4# Intersection with above: 3filters_combined = {"op": "and",- "conds": [- filters,- {"op": "must", "field": "category", "conds": ["electronics"]}- ]+ "conds": [filters, {"op": "must", "field": "category", "conds": ["electronics"]}],}self.assertEqual(self._search(filters_combined), [3])def test_multi_layer_logic(self):"""Test multi-layer logic structure (A OR (B AND (C OR D)))"""# (Category="books" OR (Category="clothing" AND (Price > 150 OR Rating > 4.2)))-+# A: Category="books" -> 7, 8# B: Category="clothing" -> 9, 10# C: Price > 150 -> 1, 2, 3, 4, 5, 6, 10@@ -294,7 +389,7 @@ class TestFilterOpsComplex(unittest.TestCase):# C OR D -> 1, 2, 3, 4, 5, 6, 7, 8, 10# B AND (C OR D) -> {9, 10} INTERSECT {1..8, 10} -> {10}# A OR (B AND ...) -> {7, 8} UNION {10} -> {7, 8, 10}-+filters = {"op": "or","conds": [@@ -307,29 +402,29 @@ class TestFilterOpsComplex(unittest.TestCase):"op": "or","conds": [{"op": "range", "field": "price", "gt": 150},- {"op": "range", "field": "rating_int", "gt": 42}- ]- }- ]- }- ]+ {"op": "range", "field": "rating_int", "gt": 42},+ ],+ },+ ],+ },+ ],}self.assertEqual(self._search(filters), [7, 8, 10])def test_mixed_type_logic(self):"""Test mixed type field filtering (String Prefix + Int Range + Logic)"""# (Tags prefix "mobile" AND Price < 5000) OR (Tags prefix "kitchen" AND Price < 500)-+# Part 1: Tags prefix "mobile" -> 1, 2# Price < 5000 -> 2, 4, 5, 6, 7, 8, 9, 10# Intersection -> 2-+# Part 2: Tags prefix "kitchen" -> 6# Price < 500 -> 6, 7, 8, 9, 10# Intersection -> 6-+# Union -> 2, 6-+filters = {"op": "or","conds": [@@ -337,17 +432,17 @@ class TestFilterOpsComplex(unittest.TestCase):"op": "and","conds": [{"op": "prefix", "field": "tags", "prefix": "mobile"},- {"op": "range", "field": "price", "lt": 5000}- ]+ {"op": "range", "field": "price", "lt": 5000},+ ],},{"op": "and","conds": [{"op": "prefix", "field": "tags", "prefix": "kitchen"},- {"op": "range", "field": "price", "lt": 500}- ]- }- ]+ {"op": "range", "field": "price", "lt": 500},+ ],+ },+ ],}self.assertEqual(self._search(filters), [2, 6])@@ -363,8 +458,8 @@ class TestFilterOpsComplex(unittest.TestCase):"conds": [{"op": "must", "field": "category", "conds": ["books"]},{"op": "must", "field": "category", "conds": ["clothing"]},- {"op": "range", "field": "price", "gt": 10000}- ]+ {"op": "range", "field": "price", "gt": 10000},+ ],}self.assertEqual(self._search(filters), [3, 7, 8, 9, 10])@@ -378,15 +473,15 @@ class TestFilterOpsComplex(unittest.TestCase):"op": "and","conds": [{"op": "must_not", "field": "category", "conds": ["electronics"]},- {"op": "range", "field": "price", "gte": 100} # Equivalent to MustNot(Price < 100)- ]+ {"op": "range", "field": "price", "gte": 100}, # Equivalent to MustNot(Price < 100)+ ],}self.assertEqual(self._search(filters), [5, 6, 9, 10])-class TestFilterOpsLifecycle(unittest.TestCase):"""Insert/update/delete and restart tests"""+def setUp(self):clean_dir(db_path_lifecycle)self.path = db_path_lifecycle@@ -431,17 +526,16 @@ class TestFilterOpsLifecycle(unittest.TestCase):def _search(self, filters, coll=None):c = coll if coll else self.collectionres = c.search_by_vector(- "idx_lifecycle",- dense_vector=[1.0, 0, 0, 0],- limit=100,- filters=filters+ "idx_lifecycle", dense_vector=[1.0, 0, 0, 0], limit=100, filters=filters)return sorted([item.id for item in res.data])def test_update_impact(self):"""Test Update impact on Filter"""# Initial state: status=active -> 1, 2- self.assertEqual(self._search({"op": "must", "field": "status", "conds": ["active"]}), [1, 2])+ self.assertEqual(+ self._search({"op": "must", "field": "status", "conds": ["active"]}), [1, 2]+ )# Update id=2 status to inactive# Update id=3 status to active@@ -452,7 +546,9 @@ class TestFilterOpsLifecycle(unittest.TestCase):self.collection.upsert_data(updates)# Verify after update: status=active -> 1, 3- self.assertEqual(self._search({"op": "must", "field": "status", "conds": ["active"]}), [1, 3])+ self.assertEqual(+ self._search({"op": "must", "field": "status", "conds": ["active"]}), [1, 3]+ )# Verify count update: count > 30 -> 3 (35)self.assertEqual(self._search({"op": "range", "field": "count", "gt": 30}), [3])@@ -479,7 +575,7 @@ class TestFilterOpsLifecycle(unittest.TestCase):# Ensure data is written# Simulate restart: release old object, reloaddel self.collection- self.collection = None # Avoid tearDown accessing deleted attribute+ self.collection = None # Avoid tearDown accessing deleted attributegc.collect()time.sleep(0.1)@@ -500,9 +596,9 @@ class TestFilterOpsLifecycle(unittest.TestCase):# tearDown will handle drop-class TestFilterOpsPath(unittest.TestCase):"""Path type Filter tests"""+def setUp(self):clean_dir("./db_test_filters_path/")self.path = "./db_test_filters_path/"@@ -547,28 +643,35 @@ class TestFilterOpsPath(unittest.TestCase):def _search(self, filters):res = self.collection.search_by_vector(- "idx_path",- dense_vector=[1.0, 0, 0, 0],- limit=100,- filters=filters+ "idx_path", dense_vector=[1.0, 0, 0, 0], limit=100, filters=filters)return sorted([item.id for item in res.data])def test_path_must(self):"""Test Must matching path prefix"""# Must /a -> /a/b/c, /a/b/d, /a/e- self.assertEqual(self._search({"op": "must", "field": "file_path", "conds": ["/a"]}), [1, 2, 3])+ self.assertEqual(+ self._search({"op": "must", "field": "file_path", "conds": ["/a"]}), [1, 2, 3]+ )# Must /a/b -> /a/b/c, /a/b/d- self.assertEqual(self._search({"op": "must", "field": "file_path", "conds": ["/a/b"]}), [1, 2])+ self.assertEqual(+ self._search({"op": "must", "field": "file_path", "conds": ["/a/b"]}), [1, 2]+ )# Must /f -> /f/g, /f/h/i- self.assertEqual(self._search({"op": "must", "field": "file_path", "conds": ["/f"]}), [4, 5])+ self.assertEqual(+ self._search({"op": "must", "field": "file_path", "conds": ["/f"]}), [4, 5]+ )def test_path_must_not(self):"""Test MustNot excluding path prefix"""# MustNot /a/b -> exclude 1, 2 -> remaining 3, 4, 5- self.assertEqual(self._search({"op": "must_not", "field": "file_path", "conds": ["/a/b"]}), [3, 4, 5])+ self.assertEqual(+ self._search({"op": "must_not", "field": "file_path", "conds": ["/a/b"]}), [3, 4, 5]+ )# MustNot /a -> exclude 1, 2, 3 -> remaining 4, 5- self.assertEqual(self._search({"op": "must_not", "field": "file_path", "conds": ["/a"]}), [4, 5])+ self.assertEqual(+ self._search({"op": "must_not", "field": "file_path", "conds": ["/a"]}), [4, 5]+ )def test_path_depth(self):"""Test path type depth parameter"""@@ -607,28 +710,28 @@ class TestFilterOpsPath(unittest.TestCase):# Must /a, depth=1# /a/e -> Yes# /a/b/c -> No (depth 2)-+self.assertEqual(- self._search({"op": "must", "field": "file_path", "conds": ["/a"], "para": "-d=1"}),- [3]+ self._search({"op": "must", "field": "file_path", "conds": ["/a"], "para": "-d=1"}), [3])# Must /a, depth=2 (default recursive all? Or specified depth)# /a/e (depth 1) -> Yes# /a/b/c (depth 2) -> Yesself.assertEqual(- self._search({"op": "must", "field": "file_path", "conds": ["/a"], "para": "-d=2"}),- [1, 2, 3]+ self._search({"op": "must", "field": "file_path", "conds": ["/a"], "para": "-d=2"}),+ [1, 2, 3],)class TestFilterOpsScale(unittest.TestCase):"""Scale tests"""+def setUp(self):clean_dir(db_path_scale)self.path = db_path_scaleself.collection = self._create_collection()- self.data_count = 500000 # 50w+ self.data_count = 500000 # 50wself._insert_large_data()self._create_index()@@ -657,35 +760,37 @@ class TestFilterOpsScale(unittest.TestCase):# SubGroup: X (even id), Y (odd id)# Score: id % 100# Tag: prefix_{id%1000}-+batch_size = 10000data = []print(f"Inserting {self.data_count} items...")start_time = time.time()-+for i in range(self.data_count):group = "A" if i < self.data_count / 2 else "B"sub_group = "X" if i % 2 == 0 else "Y"score = i % 100tag = f"tag_{i % 1000}"-- data.append({- "id": i,- "embedding": [random.random() for _ in range(4)],- "group": group,- "sub_group": sub_group,- "score": score,- "tag": tag- })-++ data.append(+ {+ "id": i,+ "embedding": [random.random() for _ in range(4)],+ "group": group,+ "sub_group": sub_group,+ "score": score,+ "tag": tag,+ }+ )+if len(data) >= batch_size:self.collection.upsert_data(data)data = []# print(f"Inserted {i+1} items")-+if data:self.collection.upsert_data(data)-+print(f"Insert finished in {time.time() - start_time:.2f}s")def _create_index(self):@@ -702,10 +807,7 @@ class TestFilterOpsScale(unittest.TestCase):def _search(self, filters, limit=10000):# For large scale test, we just check count mainlyres = self.collection.search_by_vector(- "idx_scale",- dense_vector=[0.0, 0.0, 0.0, 0.0],- limit=limit,- filters=filters+ "idx_scale", dense_vector=[0.0, 0.0, 0.0, 0.0], limit=limit, filters=filters)return [item.id for item in res.data]@@ -721,23 +823,23 @@ class TestFilterOpsScale(unittest.TestCase):"op": "and","conds": [{"op": "must", "field": "group", "conds": ["A"]},- {"op": "range", "field": "score", "gt": 90}- ]+ {"op": "range", "field": "score", "gt": 90},+ ],}# limit needs to be large enough to catch all or we verify count returned is limit# If limit < expected, we get limit. If limit > expected, we get expected.# Let's set a large limit for counting test, but verify performance# 22500 results is large. Let's just sample top 100 but rely on a smaller range for exact count verification# or verify standard limit behavior.-+# Search with smaller result set for verification# Group A AND Score = 99 (1 per 100) -> 2500 expectedfilters = {"op": "and","conds": [{"op": "must", "field": "group", "conds": ["A"]},- {"op": "must", "field": "score", "conds": [99]}- ]+ {"op": "must", "field": "score", "conds": [99]},+ ],}ids = self._search(filters, limit=5000)print(f"Filter 1 time: {time.time() - start_time:.4f}s")@@ -752,22 +854,22 @@ class TestFilterOpsScale(unittest.TestCase):# Group B (250000-499999) AND Score < 5 (0,1,2,3,4 -> 5 per 100) -> 2500 * 5 = 12500 items# Total = 137500 items. Too many to fetch all.# Let's add more conditions to reduce result set.-- # (Group A AND SubGroup X AND Tag="tag_0")++ # (Group A AND SubGroup X AND Tag="tag_0")# Tag="tag_0" -> id % 1000 == 0.# Group A (0-249999): 250 items with tag_0.# SubGroup X (Even): tag_0 implies id%1000=0 which is even. So all 250 items match X.# Result: 250 items.-+# OR-+# (Group B AND Score=0 AND SubGroup Y)# Group B (250000-499999)# Score=0 -> id % 100 == 0.# SubGroup Y (Odd) -> id is Odd.# id % 100 == 0 implies id is Even. So (Even AND Odd) -> Empty.# Result: 0 items.-+# Total expected: 250 items.print("Testing Filter 2: Complex Nested Logic")filters = {@@ -778,18 +880,18 @@ class TestFilterOpsScale(unittest.TestCase):"conds": [{"op": "must", "field": "group", "conds": ["A"]},{"op": "must", "field": "sub_group", "conds": ["X"]},- {"op": "must", "field": "tag", "conds": ["tag_0"]}- ]+ {"op": "must", "field": "tag", "conds": ["tag_0"]},+ ],},{"op": "and","conds": [{"op": "must", "field": "group", "conds": ["B"]},{"op": "must", "field": "score", "conds": [0]},- {"op": "must", "field": "sub_group", "conds": ["Y"]}- ]- }- ]+ {"op": "must", "field": "sub_group", "conds": ["Y"]},+ ],+ },+ ],}start_time = time.time()ids = self._search(filters, limit=1000)@@ -812,5 +914,6 @@ class TestFilterOpsScale(unittest.TestCase):for i in ids:self.assertEqual(i % 1000, 999)-if __name__ == '__main__':++if __name__ == "__main__":unittest.main()diff --git a/tests/vectordb/test_project_group.py b/tests/vectordb/test_project_group.pyindex 53f7ee4a3..093956e65 100644--- a/tests/vectordb/test_project_group.py+++ b/tests/vectordb/test_project_group.py@@ -3,10 +3,12 @@import osimport shutilimport unittest-from openviking.storage.vectordb.project.project_group import ProjectGroup, get_or_create_project_group++from openviking.storage.vectordb.project.project_group import get_or_create_project_groupTEST_PROJECT_ROOT = "./test_project_root"+class TestProjectGroup(unittest.TestCase):def setUp(self):if os.path.exists(TEST_PROJECT_ROOT):@@ -20,40 +22,40 @@ class TestProjectGroup(unittest.TestCase):# Path empty -> Volatilegroup = get_or_create_project_group("")self.assertTrue(group.has_project("default"))-+# Create newp1 = group.create_project("p1")self.assertIsNotNone(p1)self.assertTrue(group.has_project("p1"))-+# Listnames = group.list_projects()self.assertIn("default", names)self.assertIn("p1", names)-+# Closegroup.close()def test_persistent_group_lifecycle(self):# 1. Create and populategroup = get_or_create_project_group(TEST_PROJECT_ROOT)-+# Default should be created automaticallyself.assertTrue(group.has_project("default"))self.assertTrue(os.path.exists(os.path.join(TEST_PROJECT_ROOT, "default")))-+# Create persistent projectgroup.create_project("analytics")self.assertTrue(os.path.exists(os.path.join(TEST_PROJECT_ROOT, "analytics")))-+# Close to flush/releasegroup.close()-+# 2. Reload from diskgroup2 = get_or_create_project_group(TEST_PROJECT_ROOT)self.assertTrue(group2.has_project("default"))self.assertTrue(group2.has_project("analytics"))-+# 3. Delete projectgroup2.delete_project("analytics")self.assertFalse(group2.has_project("analytics"))@@ -66,7 +68,7 @@ class TestProjectGroup(unittest.TestCase):# project.close()# It does not seem to remove the project directory itself in `project_group.py`.# However, for robustness, we at least ensure it's gone from memory.-+group2.close()def test_duplicate_create_error(self):@@ -76,5 +78,6 @@ class TestProjectGroup(unittest.TestCase):group.create_project("dup_test")group.close()+if __name__ == "__main__":unittest.main()diff --git a/tests/vectordb/test_pydantic_validation.py b/tests/vectordb/test_pydantic_validation.pyindex 1352295ea..66d9f41e1 100644--- a/tests/vectordb/test_pydantic_validation.py+++ b/tests/vectordb/test_pydantic_validation.py@@ -1,13 +1,14 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0-import sysimport os+import sysimport unittest+from openviking.storage.vectordb.utils import validation+sys.path.append(os.getcwd())-from openviking.storage.vectordb.utils import validationclass TestPydanticValidation(unittest.TestCase):def test_valid_collection_meta(self):@@ -16,8 +17,8 @@ class TestPydanticValidation(unittest.TestCase):"Fields": [{"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},{"FieldName": "vec", "FieldType": "vector", "Dim": 128},- {"FieldName": "desc", "FieldType": "string"}- ]+ {"FieldName": "desc", "FieldType": "string"},+ ],}self.assertTrue(validation.is_valid_collection_meta_data(meta))@@ -25,41 +26,37 @@ class TestPydanticValidation(unittest.TestCase):# Missing required fieldmeta = {"Fields": []}self.assertFalse(validation.is_valid_collection_meta_data(meta))-+# Invalid dimmeta = {"CollectionName": "test",- "Fields": [{"FieldName": "vec", "FieldType": "vector", "Dim": 129}] # Not multiple of 4+ "Fields": [+ {"FieldName": "vec", "FieldType": "vector", "Dim": 129}+ ], # Not multiple of 4}self.assertFalse(validation.is_valid_collection_meta_data(meta))def test_valid_index_meta(self):meta = {"IndexName": "test_index",- "VectorIndex": {- "IndexType": "flat",- "Distance": "L2",- "Quant": "float"- }+ "VectorIndex": {"IndexType": "flat", "Distance": "L2", "Quant": "float"},}fields_meta = {}self.assertTrue(validation.is_valid_index_meta_data(meta, fields_meta))def test_fix_collection_meta(self):- meta = {- "CollectionName": "test",- "Fields": [- {"FieldName": "text", "FieldType": "string"}- ]- }+ meta = {"CollectionName": "test", "Fields": [{"FieldName": "text", "FieldType": "string"}]}fixed = validation.fix_collection_meta(meta)-+# Check if AUTO_ID was added- has_auto_id = any(f["FieldName"] == "AUTO_ID" and f["IsPrimaryKey"] for f in fixed["Fields"])+ has_auto_id = any(+ f["FieldName"] == "AUTO_ID" and f["IsPrimaryKey"] for f in fixed["Fields"]+ )self.assertTrue(has_auto_id)-+# Check _FieldID assignmentself.assertIn("_FieldID", fixed["Fields"][0])-if __name__ == '__main__':++if __name__ == "__main__":unittest.main()diff --git a/tests/vectordb/test_recall.py b/tests/vectordb/test_recall.pyindex 9501aa86a..cdc5c39e7 100644--- a/tests/vectordb/test_recall.py+++ b/tests/vectordb/test_recall.py@@ -1,21 +1,24 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0-import unittest-import shutilimport random-import math-from typing import List, Tuple+import shutil+import unittest+from typing import List+from openviking.storage.vectordb.collection.local_collection import get_or_create_local_collection# Test data pathTEST_DB_PATH = "./test_recall_collection/"+def calculate_l2_distance(v1: List[float], v2: List[float]) -> float:return sum((a - b) ** 2 for a, b in zip(v1, v2))+def calculate_ip_distance(v1: List[float], v2: List[float]) -> float:return sum(a * b for a, b in zip(v1, v2))+class TestRecall(unittest.TestCase):"""Test vector recall quality"""@@ -41,7 +44,7 @@ class TestRecall(unittest.TestCase):def test_exact_match_recall(self):"""Test if the exact vector is recalled at rank 1"""print("\n=== Test: Exact Match Recall ===")-+dim = 64meta_data = {"CollectionName": "test_exact_match",@@ -73,26 +76,28 @@ class TestRecall(unittest.TestCase):{"IndexName": "idx_flat","VectorIndex": {"IndexType": "flat", "Distance": "l2"},- }+ },)# Query with an existing vectortarget_idx = 500query_vec = vectors[target_idx]-+result = collection.search_by_vector("idx_flat", dense_vector=query_vec, limit=10)-+self.assertTrue(len(result.data) > 0)# The first result should be the vector itself (id=500)- # Note: Depending on floating point precision, distance might not be exactly 0.0,+ # Note: Depending on floating point precision, distance might not be exactly 0.0,# but it should be the closest.- self.assertEqual(result.data[0].id, target_idx, "The top result should be the query vector itself")+ self.assertEqual(+ result.data[0].id, target_idx, "The top result should be the query vector itself"+ )print("β Exact match verified")def test_l2_recall_topk(self):"""Test Top-K recall for L2 distance"""print("\n=== Test: Top-K Recall (L2) ===")-+dim = 32total_records = 500meta_data = {@@ -115,53 +120,55 @@ class TestRecall(unittest.TestCase):vec = [random.uniform(0, 1) for _ in range(dim)]vectors.append(vec)data.append({"id": i, "vector": vec})-+collection.upsert_data(data)-+collection.create_index("idx_l2",{"IndexName": "idx_l2","VectorIndex": {"IndexType": "flat", "Distance": "l2"},- }+ },)# Generate a query vectorquery_vec = [random.uniform(0, 1) for _ in range(dim)]-+# Calculate Ground Truth# (distance, id)distances = []for i, vec in enumerate(vectors):dist = calculate_l2_distance(query_vec, vec)distances.append((dist, i))-+# Sort by distance ascending (L2)distances.sort(key=lambda x: x[0])ground_truth_ids = [x[1] for x in distances[:10]]-+# Searchresult = collection.search_by_vector("idx_l2", dense_vector=query_vec, limit=10)result_ids = [item.id for item in result.data]-+print(f"Ground Truth IDs: {ground_truth_ids}")print(f"Search Result IDs: {result_ids}")-+# Calculate Recall@10intersection = set(ground_truth_ids) & set(result_ids)recall = len(intersection) / 10.0print(f"Recall@10: {recall}")-+self.assertEqual(recall, 1.0, "Recall@10 for Flat index should be 1.0")-+# Verify order matches- self.assertEqual(result_ids, ground_truth_ids, "Result order should match ground truth for Flat index")+ self.assertEqual(+ result_ids, ground_truth_ids, "Result order should match ground truth for Flat index"+ )print("β L2 Recall verified")def test_ip_recall_topk(self):"""Test Top-K recall for Inner Product (IP) distance"""print("\n=== Test: Top-K Recall (IP) ===")-+dim = 32total_records = 500meta_data = {@@ -181,51 +188,54 @@ class TestRecall(unittest.TestCase):vectors = []data = []for i in range(total_records):- # Normalize vectors for IP to behave like Cosine Similarity if needed,- # but IP itself is just dot product.+ # Normalize vectors for IP to behave like Cosine Similarity if needed,+ # but IP itself is just dot product.vec = [random.uniform(-1, 1) for _ in range(dim)]vectors.append(vec)data.append({"id": i, "vector": vec})-+collection.upsert_data(data)-+collection.create_index("idx_ip",{"IndexName": "idx_ip","VectorIndex": {"IndexType": "flat", "Distance": "ip"},- }+ },)# Generate a query vectorquery_vec = [random.uniform(-1, 1) for _ in range(dim)]-+# Calculate Ground Truth# (score, id)scores = []for i, vec in enumerate(vectors):score = calculate_ip_distance(query_vec, vec)scores.append((score, i))-+# Sort by score descending (IP)scores.sort(key=lambda x: x[0], reverse=True)ground_truth_ids = [x[1] for x in scores[:10]]-+# Searchresult = collection.search_by_vector("idx_ip", dense_vector=query_vec, limit=10)result_ids = [item.id for item in result.data]-+print(f"Ground Truth IDs: {ground_truth_ids}")print(f"Search Result IDs: {result_ids}")-+# Calculate Recall@10intersection = set(ground_truth_ids) & set(result_ids)recall = len(intersection) / 10.0print(f"Recall@10: {recall}")-+self.assertEqual(recall, 1.0, "Recall@10 for Flat index should be 1.0")- self.assertEqual(result_ids, ground_truth_ids, "Result order should match ground truth for Flat index")+ self.assertEqual(+ result_ids, ground_truth_ids, "Result order should match ground truth for Flat index"+ )print("β IP Recall verified")+if __name__ == "__main__":unittest.main()diff --git a/uv.lock b/uv.lockindex 708fa5c89..9729818a3 100644--- a/uv.lock+++ b/uv.lock@@ -113,103 +113,6 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" },]-[[package]]-name = "black"-version = "25.11.0"-source = { registry = "https://pypi.org/simple" }-resolution-markers = [- "python_full_version < '3.10'",-]-dependencies = [- { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },- { name = "mypy-extensions", marker = "python_full_version < '3.10'" },- { name = "packaging", marker = "python_full_version < '3.10'" },- { name = "pathspec", marker = "python_full_version < '3.10'" },- { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },- { name = "pytokens", marker = "python_full_version < '3.10'" },- { name = "tomli", marker = "python_full_version < '3.10'" },- { name = "typing-extensions", marker = "python_full_version < '3.10'" },-]-sdist = { url = "https://files.pythonhosted.org/packages/8c/ad/33adf4708633d047950ff2dfdea2e215d84ac50ef95aff14a614e4b6e9b2/black-25.11.0.tar.gz", hash = "sha256:9a323ac32f5dc75ce7470501b887250be5005a01602e931a15e45593f70f6e08", size = 655669, upload-time = "2025-11-10T01:53:50.558Z" }-wheels = [- { url = "https://files.pythonhosted.org/packages/b3/d2/6caccbc96f9311e8ec3378c296d4f4809429c43a6cd2394e3c390e86816d/black-25.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ec311e22458eec32a807f029b2646f661e6859c3f61bc6d9ffb67958779f392e", size = 1743501, upload-time = "2025-11-10T01:59:06.202Z" },- { url = "https://files.pythonhosted.org/packages/69/35/b986d57828b3f3dccbf922e2864223197ba32e74c5004264b1c62bc9f04d/black-25.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1032639c90208c15711334d681de2e24821af0575573db2810b0763bcd62e0f0", size = 1597308, upload-time = "2025-11-10T01:57:58.633Z" },- { url = "https://files.pythonhosted.org/packages/39/8e/8b58ef4b37073f52b64a7b2dd8c9a96c84f45d6f47d878d0aa557e9a2d35/black-25.11.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0c0f7c461df55cf32929b002335883946a4893d759f2df343389c4396f3b6b37", size = 1656194, upload-time = "2025-11-10T01:57:10.909Z" },- { url = "https://files.pythonhosted.org/packages/8d/30/9c2267a7955ecc545306534ab88923769a979ac20a27cf618d370091e5dd/black-25.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:f9786c24d8e9bd5f20dc7a7f0cdd742644656987f6ea6947629306f937726c03", size = 1347996, upload-time = "2025-11-10T01:57:22.391Z" },- { url = "https://files.pythonhosted.org/packages/c4/62/d304786b75ab0c530b833a89ce7d997924579fb7484ecd9266394903e394/black-25.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:895571922a35434a9d8ca67ef926da6bc9ad464522a5fe0db99b394ef1c0675a", size = 1727891, upload-time = "2025-11-10T02:01:40.507Z" },- { url = "https://files.pythonhosted.org/packages/82/5d/ffe8a006aa522c9e3f430e7b93568a7b2163f4b3f16e8feb6d8c3552761a/black-25.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cb4f4b65d717062191bdec8e4a442539a8ea065e6af1c4f4d36f0cdb5f71e170", size = 1581875, upload-time = "2025-11-10T01:57:51.192Z" },- { url = "https://files.pythonhosted.org/packages/cb/c8/7c8bda3108d0bb57387ac41b4abb5c08782b26da9f9c4421ef6694dac01a/black-25.11.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d81a44cbc7e4f73a9d6ae449ec2317ad81512d1e7dce7d57f6333fd6259737bc", size = 1642716, upload-time = "2025-11-10T01:56:51.589Z" },- { url = "https://files.pythonhosted.org/packages/34/b9/f17dea34eecb7cc2609a89627d480fb6caea7b86190708eaa7eb15ed25e7/black-25.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:7eebd4744dfe92ef1ee349dc532defbf012a88b087bb7ddd688ff59a447b080e", size = 1352904, upload-time = "2025-11-10T01:59:26.252Z" },- { url = "https://files.pythonhosted.org/packages/7f/12/5c35e600b515f35ffd737da7febdb2ab66bb8c24d88560d5e3ef3d28c3fd/black-25.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:80e7486ad3535636657aa180ad32a7d67d7c273a80e12f1b4bfa0823d54e8fac", size = 1772831, upload-time = "2025-11-10T02:03:47Z" },- { url = "https://files.pythonhosted.org/packages/1a/75/b3896bec5a2bb9ed2f989a970ea40e7062f8936f95425879bbe162746fe5/black-25.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6cced12b747c4c76bc09b4db057c319d8545307266f41aaee665540bc0e04e96", size = 1608520, upload-time = "2025-11-10T01:58:46.895Z" },- { url = "https://files.pythonhosted.org/packages/f3/b5/2bfc18330eddbcfb5aab8d2d720663cd410f51b2ed01375f5be3751595b0/black-25.11.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6cb2d54a39e0ef021d6c5eef442e10fd71fcb491be6413d083a320ee768329dd", size = 1682719, upload-time = "2025-11-10T01:56:55.24Z" },- { url = "https://files.pythonhosted.org/packages/96/fb/f7dc2793a22cdf74a72114b5ed77fe3349a2e09ef34565857a2f917abdf2/black-25.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:ae263af2f496940438e5be1a0c1020e13b09154f3af4df0835ea7f9fe7bfa409", size = 1362684, upload-time = "2025-11-10T01:57:07.639Z" },- { url = "https://files.pythonhosted.org/packages/ad/47/3378d6a2ddefe18553d1115e36aea98f4a90de53b6a3017ed861ba1bd3bc/black-25.11.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0a1d40348b6621cc20d3d7530a5b8d67e9714906dfd7346338249ad9c6cedf2b", size = 1772446, upload-time = "2025-11-10T02:02:16.181Z" },- { url = "https://files.pythonhosted.org/packages/ba/4b/0f00bfb3d1f7e05e25bfc7c363f54dc523bb6ba502f98f4ad3acf01ab2e4/black-25.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:51c65d7d60bb25429ea2bf0731c32b2a2442eb4bd3b2afcb47830f0b13e58bfd", size = 1607983, upload-time = "2025-11-10T02:02:52.502Z" },- { url = "https://files.pythonhosted.org/packages/99/fe/49b0768f8c9ae57eb74cc10a1f87b4c70453551d8ad498959721cc345cb7/black-25.11.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:936c4dd07669269f40b497440159a221ee435e3fddcf668e0c05244a9be71993", size = 1682481, upload-time = "2025-11-10T01:57:12.35Z" },- { url = "https://files.pythonhosted.org/packages/55/17/7e10ff1267bfa950cc16f0a411d457cdff79678fbb77a6c73b73a5317904/black-25.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:f42c0ea7f59994490f4dccd64e6b2dd49ac57c7c84f38b8faab50f8759db245c", size = 1363869, upload-time = "2025-11-10T01:58:24.608Z" },- { url = "https://files.pythonhosted.org/packages/67/c0/cc865ce594d09e4cd4dfca5e11994ebb51604328489f3ca3ae7bb38a7db5/black-25.11.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:35690a383f22dd3e468c85dc4b915217f87667ad9cce781d7b42678ce63c4170", size = 1771358, upload-time = "2025-11-10T02:03:33.331Z" },- { url = "https://files.pythonhosted.org/packages/37/77/4297114d9e2fd2fc8ab0ab87192643cd49409eb059e2940391e7d2340e57/black-25.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dae49ef7369c6caa1a1833fd5efb7c3024bb7e4499bf64833f65ad27791b1545", size = 1612902, upload-time = "2025-11-10T01:59:33.382Z" },- { url = "https://files.pythonhosted.org/packages/de/63/d45ef97ada84111e330b2b2d45e1dd163e90bd116f00ac55927fb6bf8adb/black-25.11.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bd4a22a0b37401c8e492e994bce79e614f91b14d9ea911f44f36e262195fdda", size = 1680571, upload-time = "2025-11-10T01:57:04.239Z" },- { url = "https://files.pythonhosted.org/packages/ff/4b/5604710d61cdff613584028b4cb4607e56e148801ed9b38ee7970799dab6/black-25.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:aa211411e94fdf86519996b7f5f05e71ba34835d8f0c0f03c00a26271da02664", size = 1382599, upload-time = "2025-11-10T01:57:57.427Z" },- { url = "https://files.pythonhosted.org/packages/d5/9a/5b2c0e3215fe748fcf515c2dd34658973a1210bf610e24de5ba887e4f1c8/black-25.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a3bb5ce32daa9ff0605d73b6f19da0b0e6c1f8f2d75594db539fdfed722f2b06", size = 1743063, upload-time = "2025-11-10T02:02:43.175Z" },- { url = "https://files.pythonhosted.org/packages/a1/20/245164c6efc27333409c62ba54dcbfbe866c6d1957c9a6c0647786e950da/black-25.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9815ccee1e55717fe9a4b924cae1646ef7f54e0f990da39a34fc7b264fcf80a2", size = 1596867, upload-time = "2025-11-10T02:00:17.157Z" },- { url = "https://files.pythonhosted.org/packages/ca/6f/1a3859a7da205f3d50cf3a8bec6bdc551a91c33ae77a045bb24c1f46ab54/black-25.11.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92285c37b93a1698dcbc34581867b480f1ba3a7b92acf1fe0467b04d7a4da0dc", size = 1655678, upload-time = "2025-11-10T01:57:09.028Z" },- { url = "https://files.pythonhosted.org/packages/56/1a/6dec1aeb7be90753d4fcc273e69bc18bfd34b353223ed191da33f7519410/black-25.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:43945853a31099c7c0ff8dface53b4de56c41294fa6783c0441a8b1d9bf668bc", size = 1347452, upload-time = "2025-11-10T01:57:01.871Z" },- { url = "https://files.pythonhosted.org/packages/00/5d/aed32636ed30a6e7f9efd6ad14e2a0b0d687ae7c8c7ec4e4a557174b895c/black-25.11.0-py3-none-any.whl", hash = "sha256:e3f562da087791e96cefcd9dda058380a442ab322a02e222add53736451f604b", size = 204918, upload-time = "2025-11-10T01:53:48.917Z" },-]--[[package]]-name = "black"-version = "25.12.0"-source = { registry = "https://pypi.org/simple" }-resolution-markers = [- "python_full_version >= '3.14'",- "python_full_version == '3.13.*'",- "python_full_version == '3.12.*'",- "python_full_version == '3.11.*'",- "python_full_version == '3.10.*'",-]-dependencies = [- { name = "click", version = "8.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },- { name = "mypy-extensions", marker = "python_full_version >= '3.10'" },- { name = "packaging", marker = "python_full_version >= '3.10'" },- { name = "pathspec", marker = "python_full_version >= '3.10'" },- { name = "platformdirs", version = "4.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },- { name = "pytokens", marker = "python_full_version >= '3.10'" },- { name = "tomli", marker = "python_full_version == '3.10.*'" },- { name = "typing-extensions", marker = "python_full_version == '3.10.*'" },-]-sdist = { url = "https://files.pythonhosted.org/packages/c4/d9/07b458a3f1c525ac392b5edc6b191ff140b596f9d77092429417a54e249d/black-25.12.0.tar.gz", hash = "sha256:8d3dd9cea14bff7ddc0eb243c811cdb1a011ebb4800a5f0335a01a68654796a7", size = 659264, upload-time = "2025-12-08T01:40:52.501Z" }-wheels = [- { url = "https://files.pythonhosted.org/packages/37/d5/8d3145999d380e5d09bb00b0f7024bf0a8ccb5c07b5648e9295f02ec1d98/black-25.12.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f85ba1ad15d446756b4ab5f3044731bf68b777f8f9ac9cdabd2425b97cd9c4e8", size = 1895720, upload-time = "2025-12-08T01:46:58.197Z" },- { url = "https://files.pythonhosted.org/packages/06/97/7acc85c4add41098f4f076b21e3e4e383ad6ed0a3da26b2c89627241fc11/black-25.12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:546eecfe9a3a6b46f9d69d8a642585a6eaf348bcbbc4d87a19635570e02d9f4a", size = 1727193, upload-time = "2025-12-08T01:52:26.674Z" },- { url = "https://files.pythonhosted.org/packages/24/f0/fdf0eb8ba907ddeb62255227d29d349e8256ef03558fbcadfbc26ecfe3b2/black-25.12.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:17dcc893da8d73d8f74a596f64b7c98ef5239c2cd2b053c0f25912c4494bf9ea", size = 1774506, upload-time = "2025-12-08T01:46:25.721Z" },- { url = "https://files.pythonhosted.org/packages/e4/f5/9203a78efe00d13336786b133c6180a9303d46908a9aa72d1104ca214222/black-25.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:09524b0e6af8ba7a3ffabdfc7a9922fb9adef60fed008c7cd2fc01f3048e6e6f", size = 1416085, upload-time = "2025-12-08T01:46:06.073Z" },- { url = "https://files.pythonhosted.org/packages/ba/cc/7a6090e6b081c3316282c05c546e76affdce7bf7a3b7d2c3a2a69438bd01/black-25.12.0-cp310-cp310-win_arm64.whl", hash = "sha256:b162653ed89eb942758efeb29d5e333ca5bb90e5130216f8369857db5955a7da", size = 1226038, upload-time = "2025-12-08T01:45:29.388Z" },- { url = "https://files.pythonhosted.org/packages/60/ad/7ac0d0e1e0612788dbc48e62aef8a8e8feffac7eb3d787db4e43b8462fa8/black-25.12.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d0cfa263e85caea2cff57d8f917f9f51adae8e20b610e2b23de35b5b11ce691a", size = 1877003, upload-time = "2025-12-08T01:43:29.967Z" },- { url = "https://files.pythonhosted.org/packages/e8/dd/a237e9f565f3617a88b49284b59cbca2a4f56ebe68676c1aad0ce36a54a7/black-25.12.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1a2f578ae20c19c50a382286ba78bfbeafdf788579b053d8e4980afb079ab9be", size = 1712639, upload-time = "2025-12-08T01:52:46.756Z" },- { url = "https://files.pythonhosted.org/packages/12/80/e187079df1ea4c12a0c63282ddd8b81d5107db6d642f7d7b75a6bcd6fc21/black-25.12.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d3e1b65634b0e471d07ff86ec338819e2ef860689859ef4501ab7ac290431f9b", size = 1758143, upload-time = "2025-12-08T01:45:29.137Z" },- { url = "https://files.pythonhosted.org/packages/93/b5/3096ccee4f29dc2c3aac57274326c4d2d929a77e629f695f544e159bfae4/black-25.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:a3fa71e3b8dd9f7c6ac4d818345237dfb4175ed3bf37cd5a581dbc4c034f1ec5", size = 1420698, upload-time = "2025-12-08T01:45:53.379Z" },- { url = "https://files.pythonhosted.org/packages/7e/39/f81c0ffbc25ffbe61c7d0385bf277e62ffc3e52f5ee668d7369d9854fadf/black-25.12.0-cp311-cp311-win_arm64.whl", hash = "sha256:51e267458f7e650afed8445dc7edb3187143003d52a1b710c7321aef22aa9655", size = 1229317, upload-time = "2025-12-08T01:46:35.606Z" },- { url = "https://files.pythonhosted.org/packages/d1/bd/26083f805115db17fda9877b3c7321d08c647df39d0df4c4ca8f8450593e/black-25.12.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:31f96b7c98c1ddaeb07dc0f56c652e25bdedaac76d5b68a059d998b57c55594a", size = 1924178, upload-time = "2025-12-08T01:49:51.048Z" },- { url = "https://files.pythonhosted.org/packages/89/6b/ea00d6651561e2bdd9231c4177f4f2ae19cc13a0b0574f47602a7519b6ca/black-25.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:05dd459a19e218078a1f98178c13f861fe6a9a5f88fc969ca4d9b49eb1809783", size = 1742643, upload-time = "2025-12-08T01:49:59.09Z" },- { url = "https://files.pythonhosted.org/packages/6d/f3/360fa4182e36e9875fabcf3a9717db9d27a8d11870f21cff97725c54f35b/black-25.12.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c1f68c5eff61f226934be6b5b80296cf6939e5d2f0c2f7d543ea08b204bfaf59", size = 1800158, upload-time = "2025-12-08T01:44:27.301Z" },- { url = "https://files.pythonhosted.org/packages/f8/08/2c64830cb6616278067e040acca21d4f79727b23077633953081c9445d61/black-25.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:274f940c147ddab4442d316b27f9e332ca586d39c85ecf59ebdea82cc9ee8892", size = 1426197, upload-time = "2025-12-08T01:45:51.198Z" },- { url = "https://files.pythonhosted.org/packages/d4/60/a93f55fd9b9816b7432cf6842f0e3000fdd5b7869492a04b9011a133ee37/black-25.12.0-cp312-cp312-win_arm64.whl", hash = "sha256:169506ba91ef21e2e0591563deda7f00030cb466e747c4b09cb0a9dae5db2f43", size = 1237266, upload-time = "2025-12-08T01:45:10.556Z" },- { url = "https://files.pythonhosted.org/packages/c8/52/c551e36bc95495d2aa1a37d50566267aa47608c81a53f91daa809e03293f/black-25.12.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a05ddeb656534c3e27a05a29196c962877c83fa5503db89e68857d1161ad08a5", size = 1923809, upload-time = "2025-12-08T01:46:55.126Z" },- { url = "https://files.pythonhosted.org/packages/a0/f7/aac9b014140ee56d247e707af8db0aae2e9efc28d4a8aba92d0abd7ae9d1/black-25.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9ec77439ef3e34896995503865a85732c94396edcc739f302c5673a2315e1e7f", size = 1742384, upload-time = "2025-12-08T01:49:37.022Z" },- { url = "https://files.pythonhosted.org/packages/74/98/38aaa018b2ab06a863974c12b14a6266badc192b20603a81b738c47e902e/black-25.12.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e509c858adf63aa61d908061b52e580c40eae0dfa72415fa47ac01b12e29baf", size = 1798761, upload-time = "2025-12-08T01:46:05.386Z" },- { url = "https://files.pythonhosted.org/packages/16/3a/a8ac542125f61574a3f015b521ca83b47321ed19bb63fe6d7560f348bfe1/black-25.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:252678f07f5bac4ff0d0e9b261fbb029fa530cfa206d0a636a34ab445ef8ca9d", size = 1429180, upload-time = "2025-12-08T01:45:34.903Z" },- { url = "https://files.pythonhosted.org/packages/e6/2d/bdc466a3db9145e946762d52cd55b1385509d9f9004fec1c97bdc8debbfb/black-25.12.0-cp313-cp313-win_arm64.whl", hash = "sha256:bc5b1c09fe3c931ddd20ee548511c64ebf964ada7e6f0763d443947fd1c603ce", size = 1239350, upload-time = "2025-12-08T01:46:09.458Z" },- { url = "https://files.pythonhosted.org/packages/35/46/1d8f2542210c502e2ae1060b2e09e47af6a5e5963cb78e22ec1a11170b28/black-25.12.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0a0953b134f9335c2434864a643c842c44fba562155c738a2a37a4d61f00cad5", size = 1917015, upload-time = "2025-12-08T01:53:27.987Z" },- { url = "https://files.pythonhosted.org/packages/41/37/68accadf977672beb8e2c64e080f568c74159c1aaa6414b4cd2aef2d7906/black-25.12.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2355bbb6c3b76062870942d8cc450d4f8ac71f9c93c40122762c8784df49543f", size = 1741830, upload-time = "2025-12-08T01:54:36.861Z" },- { url = "https://files.pythonhosted.org/packages/ac/76/03608a9d8f0faad47a3af3a3c8c53af3367f6c0dd2d23a84710456c7ac56/black-25.12.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9678bd991cc793e81d19aeeae57966ee02909877cb65838ccffef24c3ebac08f", size = 1791450, upload-time = "2025-12-08T01:44:52.581Z" },- { url = "https://files.pythonhosted.org/packages/06/99/b2a4bd7dfaea7964974f947e1c76d6886d65fe5d24f687df2d85406b2609/black-25.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:97596189949a8aad13ad12fcbb4ae89330039b96ad6742e6f6b45e75ad5cfd83", size = 1452042, upload-time = "2025-12-08T01:46:13.188Z" },- { url = "https://files.pythonhosted.org/packages/b2/7c/d9825de75ae5dd7795d007681b752275ea85a1c5d83269b4b9c754c2aaab/black-25.12.0-cp314-cp314-win_arm64.whl", hash = "sha256:778285d9ea197f34704e3791ea9404cd6d07595745907dd2ce3da7a13627b29b", size = 1267446, upload-time = "2025-12-08T01:46:14.497Z" },- { url = "https://files.pythonhosted.org/packages/68/11/21331aed19145a952ad28fca2756a1433ee9308079bd03bd898e903a2e53/black-25.12.0-py3-none-any.whl", hash = "sha256:48ceb36c16dbc84062740049eef990bb2ce07598272e673c17d1a7720c71c828", size = 206191, upload-time = "2025-12-08T01:40:50.963Z" },-]-[[package]]name = "certifi"version = "2025.11.12"@@ -931,37 +834,6 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },]-[[package]]-name = "isort"-version = "6.1.0"-source = { registry = "https://pypi.org/simple" }-resolution-markers = [- "python_full_version < '3.10'",-]-dependencies = [- { name = "importlib-metadata", marker = "python_full_version < '3.10'" },-]-sdist = { url = "https://files.pythonhosted.org/packages/1e/82/fa43935523efdfcce6abbae9da7f372b627b27142c3419fcf13bf5b0c397/isort-6.1.0.tar.gz", hash = "sha256:9b8f96a14cfee0677e78e941ff62f03769a06d412aabb9e2a90487b3b7e8d481", size = 824325, upload-time = "2025-10-01T16:26:45.027Z" }-wheels = [- { url = "https://files.pythonhosted.org/packages/7f/cc/9b681a170efab4868a032631dea1e8446d8ec718a7f657b94d49d1a12643/isort-6.1.0-py3-none-any.whl", hash = "sha256:58d8927ecce74e5087aef019f778d4081a3b6c98f15a80ba35782ca8a2097784", size = 94329, upload-time = "2025-10-01T16:26:43.291Z" },-]--[[package]]-name = "isort"-version = "7.0.0"-source = { registry = "https://pypi.org/simple" }-resolution-markers = [- "python_full_version >= '3.14'",- "python_full_version == '3.13.*'",- "python_full_version == '3.12.*'",- "python_full_version == '3.11.*'",- "python_full_version == '3.10.*'",-]-sdist = { url = "https://files.pythonhosted.org/packages/63/53/4f3c058e3bace40282876f9b553343376ee687f3c35a525dc79dbd450f88/isort-7.0.0.tar.gz", hash = "sha256:5513527951aadb3ac4292a41a16cbc50dd1642432f5e8c20057d414bdafb4187", size = 805049, upload-time = "2025-10-11T13:30:59.107Z" }-wheels = [- { url = "https://files.pythonhosted.org/packages/7f/ed/e3705d6d02b4f7aea715a353c8ce193efd0b5db13e204df895d38734c244/isort-7.0.0-py3-none-any.whl", hash = "sha256:1bcabac8bc3c36c7fb7b98a76c8abb18e0f841a3ba81decac7691008592499c1", size = 94672, upload-time = "2025-10-11T13:30:57.665Z" },-]-[[package]]name = "jinja2"version = "3.1.6"@@ -1830,16 +1702,12 @@ wheels = [[[package]]name = "openviking"-version = "0.1.2"+version = "0.1.6"source = { editable = "." }dependencies = [{ name = "apscheduler" },- { name = "black", version = "25.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },- { name = "black", version = "25.12.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },{ name = "fastapi" },{ name = "httpx" },- { name = "isort", version = "6.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },- { name = "isort", version = "7.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },{ name = "json-repair", version = "0.44.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },{ name = "json-repair", version = "0.55.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },{ name = "markdownify" },@@ -1878,10 +1746,8 @@ dependencies = [[package.metadata]requires-dist = [{ name = "apscheduler", specifier = ">=3.11.0" },- { name = "black", specifier = ">=23.0.0" },{ name = "fastapi", specifier = ">=0.128.0" },{ name = "httpx", specifier = ">=0.25.0" },- { name = "isort", specifier = ">=5.12.0" },{ name = "json-repair", specifier = ">=0.25.0" },{ name = "markdownify", specifier = ">=0.11.0" },{ name = "mypy", specifier = ">=1.0.0" },@@ -2341,34 +2207,6 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/2d/71/64e9b1c7f04ae0027f788a248e6297d7fcc29571371fe7d45495a78172c0/pillow-12.1.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:75af0b4c229ac519b155028fa1be632d812a519abba9b46b20e50c6caa184f19", size = 7029809, upload-time = "2026-01-02T09:13:26.541Z" },]-[[package]]-name = "platformdirs"-version = "4.4.0"-source = { registry = "https://pypi.org/simple" }-resolution-markers = [- "python_full_version < '3.10'",-]-sdist = { url = "https://files.pythonhosted.org/packages/23/e8/21db9c9987b0e728855bd57bff6984f67952bea55d6f75e055c46b5383e8/platformdirs-4.4.0.tar.gz", hash = "sha256:ca753cf4d81dc309bc67b0ea38fd15dc97bc30ce419a7f58d13eb3bf14c4febf", size = 21634, upload-time = "2025-08-26T14:32:04.268Z" }-wheels = [- { url = "https://files.pythonhosted.org/packages/40/4b/2028861e724d3bd36227adfa20d3fd24c3fc6d52032f4a93c133be5d17ce/platformdirs-4.4.0-py3-none-any.whl", hash = "sha256:abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85", size = 18654, upload-time = "2025-08-26T14:32:02.735Z" },-]--[[package]]-name = "platformdirs"-version = "4.5.1"-source = { registry = "https://pypi.org/simple" }-resolution-markers = [- "python_full_version >= '3.14'",- "python_full_version == '3.13.*'",- "python_full_version == '3.12.*'",- "python_full_version == '3.11.*'",- "python_full_version == '3.10.*'",-]-sdist = { url = "https://files.pythonhosted.org/packages/cf/86/0248f086a84f01b37aaec0fa567b397df1a119f73c16f6c7a9aac73ea309/platformdirs-4.5.1.tar.gz", hash = "sha256:61d5cdcc6065745cdd94f0f878977f8de9437be93de97c1c12f853c9c0cdcbda", size = 21715, upload-time = "2025-12-05T13:52:58.638Z" }-wheels = [- { url = "https://files.pythonhosted.org/packages/cb/28/3bfe2fa5a7b9c46fe7e13c97bda14c895fb10fa2ebf1d0abb90e0cea7ee1/platformdirs-4.5.1-py3-none-any.whl", hash = "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31", size = 18731, upload-time = "2025-12-05T13:52:56.823Z" },-]-[[package]]name = "pluggy"version = "1.6.0"@@ -2789,15 +2627,6 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },]-[[package]]-name = "pytokens"-version = "0.3.0"-source = { registry = "https://pypi.org/simple" }-sdist = { url = "https://files.pythonhosted.org/packages/4e/8d/a762be14dae1c3bf280202ba3172020b2b0b4c537f94427435f19c413b72/pytokens-0.3.0.tar.gz", hash = "sha256:2f932b14ed08de5fcf0b391ace2642f858f1394c0857202959000b68ed7a458a", size = 17644, upload-time = "2025-11-05T13:36:35.34Z" }-wheels = [- { url = "https://files.pythonhosted.org/packages/84/25/d9db8be44e205a124f6c98bc0324b2bb149b7431c53877fc6d1038dddaf5/pytokens-0.3.0-py3-none-any.whl", hash = "sha256:95b2b5eaf832e469d141a378872480ede3f251a5a5041b8ec6e581d3ac71bbf3", size = 12195, upload-time = "2025-11-05T13:36:33.183Z" },-]-[[package]]name = "pytz"version = "2025.2"