Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

Lint code (#25)

kkkwjxa30cb2e7b9bd
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index 4e87bee00..e2d00d182 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -50,11 +50,8 @@ jobs:
- name: Install dependencies
run: uv sync
- - name: Check code formatting with black
- run: uv run black --check --diff openviking/
-
- - name: Check import sorting with isort
- run: uv run isort --check-only --diff openviking/
+ - name: Format with ruff
+ run: uv run ruff format --check openviking/
- name: Lint with ruff
run: uv run ruff check openviking/
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 000000000..e08c5c4dd
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,7 @@
+repos:
+ - repo: https://github.com/astral-sh/ruff-pre-commit
+ rev: v0.14.14
+ hooks:
+ - id: ruff
+ args: [ --fix ]
+ - id: ruff-format
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index b75fdbaa1..3b69757e8 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -140,17 +140,30 @@ We use the following tools to maintain code consistency:
| Tool | Purpose | Config |
|------|---------|--------|
-| **Black** | Code formatting | `pyproject.toml` |
-| **isort** | Import sorting | `pyproject.toml` |
-| **Ruff** | Linting | `pyproject.toml` |
+| **Ruff** | Linting, Formatting, Import sorting | `pyproject.toml` |
| **mypy** | Type checking | `pyproject.toml` |
+### Automated Checks (Recommended)
+
+We use [pre-commit](https://pre-commit.com/) to automatically run these checks before every commit. This ensures your code always meets the standards without manual effort.
+
+1. **Install pre-commit**:
+ ```bash
+ pip install pre-commit
+ ```
+
+2. **Install the git hooks**:
+ ```bash
+ pre-commit install
+ ```
+
+Now, `ruff` (check & format) will run automatically when you run `git commit`. If any check fails, it may automatically fix the file. You just need to add the changes and commit again.
+
### Running Checks
```bash
# Format code
-black openviking/
-isort openviking/
+ruff format openviking/
# Lint
ruff check openviking/
diff --git a/examples/query/add.py b/examples/query/add.py
index b4e0e9f54..bafe37249 100644
--- a/examples/query/add.py
+++ b/examples/query/add.py
@@ -2,12 +2,12 @@
"""
Add Resource - CLI tool to add documents to OpenViking database
"""
-import boring_logging_config # Configure logging (set OV_DEBUG=1 for debug mode)
+
import argparse
import json
import sys
-import os
from pathlib import Path
+
import openviking as ov
from openviking.utils.config.open_viking_config import OpenVikingConfig
@@ -23,7 +23,7 @@ def add_resource(resource_path: str, config_path: str = "./ov.conf", data_path:
"""
# Load config
print(f"πŸ“‹ Loading config from: {config_path}")
- with open(config_path, 'r') as f:
+ with open(config_path, "r") as f:
config_dict = json.load(f)
config = OpenVikingConfig.from_dict(config_dict)
@@ -37,7 +37,7 @@ def add_resource(resource_path: str, config_path: str = "./ov.conf", data_path:
print(f"πŸ“‚ Adding resource: {resource_path}")
# Check if it's a file and exists
- if not resource_path.startswith('http'):
+ if not resource_path.startswith("http"):
path = Path(resource_path).expanduser()
if not path.exists():
print(f"❌ Error: File not found: {path}")
@@ -46,8 +46,8 @@ def add_resource(resource_path: str, config_path: str = "./ov.conf", data_path:
result = client.add_resource(path=resource_path)
# Check result
- if result and 'root_uri' in result:
- root_uri = result['root_uri']
+ if result and "root_uri" in result:
+ root_uri = result["root_uri"]
print(f"βœ“ Resource added: {root_uri}\n")
# Wait for processing
@@ -58,10 +58,10 @@ def add_resource(resource_path: str, config_path: str = "./ov.conf", data_path:
print("πŸŽ‰ Resource is now searchable in the database!")
return True
- elif result and result.get('status') == 'error':
- print(f"\n⚠️ Resource had parsing issues:")
- if 'errors' in result:
- for error in result['errors'][:3]:
+ elif result and result.get("status") == "error":
+ print("\n⚠️ Resource had parsing issues:")
+ if "errors" in result:
+ for error in result["errors"][:3]:
print(f" - {error}")
print("\nπŸ’‘ Some content may still be searchable.")
return False
@@ -73,6 +73,7 @@ def add_resource(resource_path: str, config_path: str = "./ov.conf", data_path:
except Exception as e:
print(f"\n❌ Error: {e}")
import traceback
+
traceback.print_exc()
return False
@@ -107,33 +108,29 @@ Notes:
- URLs are automatically downloaded and processed
- Large files may take several minutes to process
- The resource becomes searchable after processing completes
- """
+ """,
)
parser.add_argument(
- 'resource',
- type=str,
- help='Path to file/directory or URL to add to the database'
+ "resource", type=str, help="Path to file/directory or URL to add to the database"
)
parser.add_argument(
- '--config',
- type=str,
- default='./ov.conf',
- help='Path to config file (default: ./ov.conf)'
+ "--config", type=str, default="./ov.conf", help="Path to config file (default: ./ov.conf)"
)
parser.add_argument(
- '--data',
- type=str,
- default='./data',
- help='Path to data directory (default: ./data)'
+ "--data", type=str, default="./data", help="Path to data directory (default: ./data)"
)
args = parser.parse_args()
# Expand user paths
- resource_path = str(Path(args.resource).expanduser()) if not args.resource.startswith('http') else args.resource
+ resource_path = (
+ str(Path(args.resource).expanduser())
+ if not args.resource.startswith("http")
+ else args.resource
+ )
# Add the resource
success = add_resource(resource_path, args.config, args.data)
diff --git a/examples/query/boring_logging_config.py b/examples/query/boring_logging_config.py
index 531781b4e..8d7cacc8c 100644
--- a/examples/query/boring_logging_config.py
+++ b/examples/query/boring_logging_config.py
@@ -2,63 +2,102 @@
Centralized logging configuration
Set OV_DEBUG=1 environment variable to enable debug logging
"""
+
import logging
import logging.config
import os
import warnings
# Suppress warnings
-warnings.filterwarnings('ignore')
+warnings.filterwarnings("ignore")
# Check debug mode from environment
-DEBUG = os.environ.get('OV_DEBUG') == '1'
+DEBUG = os.environ.get("OV_DEBUG") == "1"
if DEBUG:
# Debug mode - show all logs
logging.basicConfig(
- level=logging.INFO,
- format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+ level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
else:
# Production mode - aggressively suppress all logs
- logging.config.dictConfig({
- 'version': 1,
- 'disable_existing_loggers': True,
- 'formatters': {
- 'null': {
- 'format': ''
- }
- },
- 'handlers': {
- 'null': {
- 'class': 'logging.NullHandler',
+ logging.config.dictConfig(
+ {
+ "version": 1,
+ "disable_existing_loggers": True,
+ "formatters": {"null": {"format": ""}},
+ "handlers": {
+ "null": {
+ "class": "logging.NullHandler",
+ },
+ },
+ "root": {"level": "CRITICAL", "handlers": ["null"]},
+ "loggers": {
+ # Suppress all OpenViking loggers
+ "openviking": {"level": "CRITICAL", "handlers": ["null"], "propagate": False},
+ "openviking.agfs_manager": {
+ "level": "CRITICAL",
+ "handlers": ["null"],
+ "propagate": False,
+ },
+ "openviking.storage": {
+ "level": "CRITICAL",
+ "handlers": ["null"],
+ "propagate": False,
+ },
+ "openviking.storage.viking_vector_index_backend": {
+ "level": "CRITICAL",
+ "handlers": ["null"],
+ "propagate": False,
+ },
+ "openviking.storage.queuefs": {
+ "level": "CRITICAL",
+ "handlers": ["null"],
+ "propagate": False,
+ },
+ "openviking.storage.queuefs.queue_manager": {
+ "level": "CRITICAL",
+ "handlers": ["null"],
+ "propagate": False,
+ },
+ "openviking.storage.vikingdb_manager": {
+ "level": "CRITICAL",
+ "handlers": ["null"],
+ "propagate": False,
+ },
+ "openviking.storage.viking_fs": {
+ "level": "CRITICAL",
+ "handlers": ["null"],
+ "propagate": False,
+ },
+ "openviking.async_client": {
+ "level": "CRITICAL",
+ "handlers": ["null"],
+ "propagate": False,
+ },
+ "openviking.parse": {"level": "CRITICAL", "handlers": ["null"], "propagate": False},
+ "openviking.parse.parsers": {
+ "level": "CRITICAL",
+ "handlers": ["null"],
+ "propagate": False,
+ },
+ "openviking.parse.parsers.markdown": {
+ "level": "CRITICAL",
+ "handlers": ["null"],
+ "propagate": False,
+ },
+ "openviking.storage.queuefs.semantic_processor": {
+ "level": "CRITICAL",
+ "handlers": ["null"],
+ "propagate": False,
+ },
+ "apscheduler": {"level": "CRITICAL", "handlers": ["null"], "propagate": False},
},
- },
- 'root': {
- 'level': 'CRITICAL',
- 'handlers': ['null']
- },
- 'loggers': {
- # Suppress all OpenViking loggers
- 'openviking': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},
- 'openviking.agfs_manager': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},
- 'openviking.storage': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},
- 'openviking.storage.viking_vector_index_backend': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},
- 'openviking.storage.queuefs': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},
- 'openviking.storage.queuefs.queue_manager': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},
- 'openviking.storage.vikingdb_manager': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},
- 'openviking.storage.viking_fs': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},
- 'openviking.async_client': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},
- 'openviking.parse': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},
- 'openviking.parse.parsers': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},
- 'openviking.parse.parsers.markdown': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},
- 'openviking.storage.queuefs.semantic_processor': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},
- 'apscheduler': {'level': 'CRITICAL', 'handlers': ['null'], 'propagate': False},
}
- })
+ )
# Additional enforcement: set all loggers after config
- for logger_name in ['openviking', 'apscheduler']:
+ for logger_name in ["openviking", "apscheduler"]:
logger = logging.getLogger(logger_name)
logger.setLevel(logging.CRITICAL)
logger.propagate = False
diff --git a/examples/query/query.py b/examples/query/query.py
index 92875dbcd..4e98ff616 100644
--- a/examples/query/query.py
+++ b/examples/query/query.py
@@ -2,13 +2,11 @@
"""
Query - CLI tool to search and generate answers using OpenViking + LLM
"""
+
import argparse
import sys
-import textwrap
import threading
-import boring_logging_config # Configure logging (set OV_DEBUG=1 for debug mode)
-
from recipe import Recipe
from rich import box
from rich.console import Console
@@ -79,7 +77,7 @@ def query(
temperature: float = 0.7,
max_tokens: int = 2048,
score_threshold: float = 0.2,
- verbose: bool = False
+ verbose: bool = False,
):
"""
Query the database and generate an answer using LLM
@@ -111,13 +109,23 @@ def query(
try:
pipeline = Recipe(config_path=config_path, data_path=data_path)
except Exception as e:
- console.print(Panel(f"❌ Error initializing pipeline: {e}", style="bold red", padding=(0, 1)))
+ console.print(
+ Panel(f"❌ Error initializing pipeline: {e}", style="bold red", padding=(0, 1))
+ )
return False
try:
# Display question with constrained width
question_text = Text(question, style="bold yellow")
- console.print(Panel(question_text, title="βœ… Roger That", style="bold", padding=(0, 1), width=PANEL_WIDTH))
+ console.print(
+ Panel(
+ question_text,
+ title="βœ… Roger That",
+ style="bold",
+ padding=(0, 1),
+ width=PANEL_WIDTH,
+ )
+ )
console.print()
# Query with loading spinner
@@ -128,20 +136,28 @@ def query(
search_top_k=top_k,
temperature=temperature,
max_tokens=max_tokens,
- score_threshold=score_threshold
+ score_threshold=score_threshold,
+ )
+ answer_text = Text(result["answer"], style="white")
+ console.print(
+ Panel(
+ answer_text,
+ title="πŸ” Check This Out",
+ style="bold bright_cyan",
+ padding=(1, 1),
+ width=PANEL_WIDTH,
+ )
)
- answer_text = Text(result['answer'], style="white")
- console.print(Panel(answer_text, title="πŸ” Check This Out", style="bold bright_cyan", padding=(1, 1), width=PANEL_WIDTH))
console.print()
# Show sources
- if result['context']:
+ if result["context"]:
sources_table = Table(
title=f"πŸ“š Sources ({len(result['context'])} documents)",
box=box.ROUNDED,
show_header=True,
header_style="bold magenta",
- title_style="bold magenta"
+ title_style="bold magenta",
)
sources_table.add_column("#", style="cyan", width=4)
sources_table.add_column("File", style="bold white")
@@ -151,26 +167,38 @@ def query(
sources_table.add_column("URI", style="dim")
sources_table.add_column("Preview", style="dim")
- for i, ctx in enumerate(result['context'], 1):
- uri_parts = ctx['uri'].split('/')
- filename = uri_parts[-1] if uri_parts else ctx['uri']
+ for i, ctx in enumerate(result["context"], 1):
+ uri_parts = ctx["uri"].split("/")
+ filename = uri_parts[-1] if uri_parts else ctx["uri"]
score_text = Text(f"{ctx['score']:.4f}", style="bold green")
if verbose:
- preview = ctx['content'][:100] + "..." if len(ctx['content']) > 100 else ctx['content']
- sources_table.add_row(str(i), filename, score_text, ctx['uri'], preview)
+ preview = (
+ ctx["content"][:100] + "..."
+ if len(ctx["content"]) > 100
+ else ctx["content"]
+ )
+ sources_table.add_row(str(i), filename, score_text, ctx["uri"], preview)
else:
sources_table.add_row(str(i), filename, score_text)
console.print(sources_table)
else:
- console.print(Panel("⚠️ No relevant sources found", style="yellow", padding=(0, 1), width=PANEL_WIDTH))
+ console.print(
+ Panel(
+ "⚠️ No relevant sources found",
+ style="yellow",
+ padding=(0, 1),
+ width=PANEL_WIDTH,
+ )
+ )
return True
except Exception as e:
console.print(Panel(f"❌ Error during query: {e}", style="bold red", padding=(0, 1)))
import traceback
+
traceback.print_exc()
return False
@@ -216,68 +244,54 @@ Score Threshold Guide:
0.0-0.1 β†’ Very permissive, includes low relevance results
0.2 β†’ Balanced (default)
0.3-0.5 β†’ Strict, only highly relevant results
- """
+ """,
)
- parser.add_argument(
- 'question',
- type=str,
- help='Your question to ask the LLM'
- )
+ parser.add_argument("question", type=str, help="Your question to ask the LLM")
parser.add_argument(
- '--config',
- type=str,
- default='./ov.conf',
- help='Path to config file (default: ./ov.conf)'
+ "--config", type=str, default="./ov.conf", help="Path to config file (default: ./ov.conf)"
)
parser.add_argument(
- '--data',
- type=str,
- default='./data',
- help='Path to data directory (default: ./data)'
+ "--data", type=str, default="./data", help="Path to data directory (default: ./data)"
)
parser.add_argument(
- '--top-k',
+ "--top-k",
type=int,
default=5,
- help='Number of search results to use as context (default: 5)'
+ help="Number of search results to use as context (default: 5)",
)
parser.add_argument(
- '--temperature',
- type=float,
- default=0.7,
- help='LLM temperature 0.0-1.0 (default: 0.7)'
+ "--temperature", type=float, default=0.7, help="LLM temperature 0.0-1.0 (default: 0.7)"
)
parser.add_argument(
- '--max-tokens',
- type=int,
- default=2048,
- help='Maximum tokens to generate (default: 2048)'
+ "--max-tokens", type=int, default=2048, help="Maximum tokens to generate (default: 2048)"
)
parser.add_argument(
- '--score-threshold',
+ "--score-threshold",
type=float,
default=0.2,
- help='Minimum relevance score for search results (default: 0.2)'
+ help="Minimum relevance score for search results (default: 0.2)",
)
- parser.add_argument(
- '--verbose', '-v',
- action='store_true',
- help='Show detailed information'
- )
+ parser.add_argument("--verbose", "-v", action="store_true", help="Show detailed information")
args = parser.parse_args()
# Validate temperature
if not 0.0 <= args.temperature <= 1.0:
- console.print(Panel("❌ Error: Temperature must be between 0.0 and 1.0", style="bold red", padding=(0, 1)))
+ console.print(
+ Panel(
+ "❌ Error: Temperature must be between 0.0 and 1.0",
+ style="bold red",
+ padding=(0, 1),
+ )
+ )
sys.exit(1)
# Validate top-k
@@ -286,7 +300,13 @@ Score Threshold Guide:
sys.exit(1)
if not 0.0 <= args.score_threshold <= 1.0:
- console.print(Panel("❌ Error: score-threshold must be between 0.0 and 1.0", style="bold red", padding=(0, 1)))
+ console.print(
+ Panel(
+ "❌ Error: score-threshold must be between 0.0 and 1.0",
+ style="bold red",
+ padding=(0, 1),
+ )
+ )
sys.exit(1)
# Run query
@@ -298,7 +318,7 @@ Score Threshold Guide:
temperature=args.temperature,
max_tokens=args.max_tokens,
score_threshold=args.score_threshold,
- verbose=args.verbose
+ verbose=args.verbose,
)
sys.exit(0 if success else 1)
diff --git a/examples/query/recipe.py b/examples/query/recipe.py
index e2631922c..2441cf1ea 100644
--- a/examples/query/recipe.py
+++ b/examples/query/recipe.py
@@ -3,12 +3,14 @@
RAG Pipeline - Retrieval-Augmented Generation using OpenViking + LLM
Focused on querying and answer generation, not resource management
"""
-import boring_logging_config # Configure logging (set OV_DEBUG=1 for debug mode)
-import openviking as ov
+
import json
+from typing import Any, Dict, List, Optional
+
import requests
+
+import openviking as ov
from openviking.utils.config.open_viking_config import OpenVikingConfig
-from typing import Optional, List, Dict, Any
class Recipe:
@@ -30,21 +32,27 @@ class Recipe:
data_path: Path to OpenViking data directory
"""
# Load configuration
- with open(config_path, 'r') as f:
+ with open(config_path, "r") as f:
self.config_dict = json.load(f)
# Extract LLM config
- self.vlm_config = self.config_dict.get('vlm', {})
- self.api_base = self.vlm_config.get('api_base')
- self.api_key = self.vlm_config.get('api_key')
- self.model = self.vlm_config.get('model')
+ self.vlm_config = self.config_dict.get("vlm", {})
+ self.api_base = self.vlm_config.get("api_base")
+ self.api_key = self.vlm_config.get("api_key")
+ self.model = self.vlm_config.get("model")
# Initialize OpenViking client
config = OpenVikingConfig.from_dict(self.config_dict)
self.client = ov.SyncOpenViking(path=data_path, config=config)
self.client.initialize()
- def search(self, query: str, top_k: int = 3, target_uri: Optional[str] = None, score_threshold: float = 0.2) -> List[Dict[str, Any]]:
+ def search(
+ self,
+ query: str,
+ top_k: int = 3,
+ target_uri: Optional[str] = None,
+ score_threshold: float = 0.2,
+ ) -> List[Dict[str, Any]]:
"""
Search for relevant content using semantic search
@@ -69,24 +77,30 @@ class Recipe:
try:
# Try to read the resource
content = self.client.read(resource.uri)
- search_results.append({
- 'uri': resource.uri,
- 'score': resource.score,
- 'content': content[:1000] # Limit content length
- })
- print(f" {i+1}. {resource.uri} (score: {resource.score:.4f})")
+ search_results.append(
+ {
+ "uri": resource.uri,
+ "score": resource.score,
+ "content": content[:1000], # Limit content length
+ }
+ )
+ print(f" {i + 1}. {resource.uri} (score: {resource.score:.4f})")
except Exception as e:
# Handle directories - read their abstract instead
if "is a directory" in str(e):
try:
abstract = self.client.abstract(resource.uri)
- search_results.append({
- 'uri': resource.uri,
- 'score': resource.score,
- 'content': f"[Directory Abstract] {abstract[:1000]}"
- })
- print(f" {i+1}. {resource.uri} (score: {resource.score:.4f}) [directory]")
+ search_results.append(
+ {
+ "uri": resource.uri,
+ "score": resource.score,
+ "content": f"[Directory Abstract] {abstract[:1000]}",
+ }
+ )
+ print(
+ f" {i + 1}. {resource.uri} (score: {resource.score:.4f}) [directory]"
+ )
except:
# Skip if we can't get abstract
continue
@@ -110,21 +124,13 @@ class Recipe:
"""
url = f"{self.api_base}/chat/completions"
- headers = {
- "Content-Type": "application/json",
- "Authorization": f"Bearer {self.api_key}"
- }
+ headers = {"Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}"}
payload = {
"model": self.model,
- "messages": [
- {
- "role": "user",
- "content": prompt
- }
- ],
+ "messages": [{"role": "user", "content": prompt}],
"temperature": temperature,
- "max_tokens": max_tokens
+ "max_tokens": max_tokens,
}
print(f"πŸ€– Calling LLM: {self.model}")
@@ -132,17 +138,19 @@ class Recipe:
response.raise_for_status()
result = response.json()
- answer = result['choices'][0]['message']['content']
+ answer = result["choices"][0]["message"]["content"]
return answer
- def query(self,
- user_query: str,
- search_top_k: int = 3,
- temperature: float = 0.7,
- max_tokens: int = 2048,
- system_prompt: Optional[str] = None,
- score_threshold: float = 0.2) -> Dict[str, Any]:
+ def query(
+ self,
+ user_query: str,
+ search_top_k: int = 3,
+ temperature: float = 0.7,
+ max_tokens: int = 2048,
+ system_prompt: Optional[str] = None,
+ score_threshold: float = 0.2,
+ ) -> Dict[str, Any]:
"""
Full RAG pipeline: search β†’ retrieve β†’ generate
@@ -158,20 +166,24 @@ class Recipe:
Dictionary with answer, context, and metadata
"""
# Step 1: Search for relevant content
- search_results = self.search(user_query, top_k=search_top_k, score_threshold=score_threshold)
+ search_results = self.search(
+ user_query, top_k=search_top_k, score_threshold=score_threshold
+ )
if not search_results:
return {
- 'answer': "I couldn't find any relevant information to answer your question.",
- 'context': [],
- 'query': user_query
+ "answer": "I couldn't find any relevant information to answer your question.",
+ "context": [],
+ "query": user_query,
}
# Step 2: Build context from search results
- context_text = "\n\n".join([
- f"[Source {i+1}] (relevance: {r['score']:.4f})\n{r['content']}"
- for i, r in enumerate(search_results)
- ])
+ context_text = "\n\n".join(
+ [
+ f"[Source {i + 1}] (relevance: {r['score']:.4f})\n{r['content']}"
+ for i, r in enumerate(search_results)
+ ]
+ )
# Step 3: Build the prompt
if system_prompt:
@@ -188,12 +200,7 @@ class Recipe:
answer = self.call_llm(prompt, temperature=temperature, max_tokens=max_tokens)
# Return full result
- return {
- 'answer': answer,
- 'context': search_results,
- 'query': user_query,
- 'prompt': prompt
- }
+ return {"answer": answer, "context": search_results, "query": user_query, "prompt": prompt}
def close(self):
"""Clean up resources"""
diff --git a/examples/quick_start.py b/examples/quick_start.py
index 4dd5ff45f..2e154d430 100644
--- a/examples/quick_start.py
+++ b/examples/quick_start.py
@@ -6,24 +6,26 @@ try:
client.initialize()
# Add resource (URL, file, or directory)
- res = client.add_resource(path="https://raw.githubusercontent.com/volcengine/MineContext/refs/heads/main/README_zh.md")
- root_uri = res['root_uri']
- res = client.ls(root_uri) # Explore resource tree
+ res = client.add_resource(
+ path="https://raw.githubusercontent.com/volcengine/MineContext/refs/heads/main/README_zh.md"
+ )
+ root_uri = res["root_uri"]
+ res = client.ls(root_uri) # Explore resource tree
print(f"Directory structure:\n{res}\n")
- res = client.glob(pattern="**/*.md", uri=root_uri) # use glob to find markdown files
- if res['matches']:
- content = client.read(res['matches'][0])
+ res = client.glob(pattern="**/*.md", uri=root_uri) # use glob to find markdown files
+ if res["matches"]:
+ content = client.read(res["matches"][0])
print(f"Content preview: {content[:200]}...\n")
print(" Wait for semantic processing...")
client.wait_processed()
- abstract = client.abstract(root_uri) # Get abstract
- overview = client.overview(root_uri) # Get overview
+ abstract = client.abstract(root_uri) # Get abstract
+ overview = client.overview(root_uri) # Get overview
print(f"Abstract:\n{abstract}\n\nOverview:\n{overview}\n")
-
- results = client.find("what is openviking", target_uri=root_uri) # Semantic search
+
+ results = client.find("what is openviking", target_uri=root_uri) # Semantic search
print("Search results:")
for r in results.resources:
print(f" {r.uri} (score: {r.score:.4f})")
@@ -31,4 +33,4 @@ try:
client.close()
except Exception as e:
- print(f"Error: {e}")
\ No newline at end of file
+ print(f"Error: {e}")
diff --git a/openviking/__init__.py b/openviking/__init__.py
index 21557d911..3fc9536e8 100644
--- a/openviking/__init__.py
+++ b/openviking/__init__.py
@@ -6,7 +6,7 @@ OpenViking - An Agent-native context database
Data in, Context out.
"""
-from openviking.client import SyncOpenViking, AsyncOpenViking
+from openviking.client import AsyncOpenViking, SyncOpenViking
from openviking.session import Session
OpenViking = SyncOpenViking
diff --git a/openviking/agfs_manager.py b/openviking/agfs_manager.py
index 1e05d1155..6898e5bd3 100644
--- a/openviking/agfs_manager.py
+++ b/openviking/agfs_manager.py
@@ -8,12 +8,15 @@ import socket
import subprocess
import time
from pathlib import Path
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
import yaml
from openviking.utils import get_logger
+if TYPE_CHECKING:
+ from openviking.utils.config.agfs_config import AGFSConfig
+
logger = get_logger(__name__)
@@ -80,7 +83,6 @@ class AGFSManager:
atexit.register(self.stop)
-
@property
def vikingfs_path(self) -> Path:
"""AGFS LocalFS data directory."""
@@ -211,7 +213,7 @@ class AGFSManager:
try:
resp = requests.get(f"{self.url}/api/v1/health", timeout=0.5)
if resp.status_code == 200:
- logger.info(f"[AGFSManager] AGFS is ready")
+ logger.info("[AGFSManager] AGFS is ready")
return
except requests.RequestException as e:
logger.debug(f"[AGFSManager] Health check failed: {e}")
@@ -219,7 +221,9 @@ class AGFSManager:
time.sleep(0.1)
# Timeout, try reading output
- logger.error(f"[AGFSManager] Timeout after {timeout}s, process still running: {self.process.poll() is None}")
+ logger.error(
+ f"[AGFSManager] Timeout after {timeout}s, process still running: {self.process.poll() is None}"
+ )
raise TimeoutError(f"AGFS failed to start within {timeout}s")
def stop(self) -> None:
@@ -241,4 +245,3 @@ class AGFSManager:
def is_running(self) -> bool:
"""Check if AGFS is running."""
return self.process is not None and self.process.poll() is None
-
diff --git a/openviking/async_client.py b/openviking/async_client.py
index 8a8975f0a..e1987bc17 100644
--- a/openviking/async_client.py
+++ b/openviking/async_client.py
@@ -10,29 +10,23 @@ from typing import Any, Dict, List, Optional
# Local imports - Core modules
from openviking.agfs_manager import AGFSManager
-from openviking.utils.config import (
- OpenVikingConfig,
- get_openviking_config,
-)
-from openviking.utils.uri import VikingURI
-from openviking.utils.config.open_viking_config import initialize_openviking_config
from openviking.core.directories import DirectoryInitializer
from openviking.session import Session
from openviking.session.compressor import SessionCompressor
from openviking.storage import VikingDBManager
from openviking.storage.collection_schemas import init_context_collection
-from openviking.storage.queuefs import get_queue_manager
+from openviking.storage.local_fs import export_ovpack as local_export_ovpack
+from openviking.storage.local_fs import import_ovpack as local_import_ovpack
from openviking.storage.observers import QueueObserver, VikingDBObserver, VLMObserver
-from openviking.storage.local_fs import (
- export_ovpack as local_export_ovpack,
- import_ovpack as local_import_ovpack,
-)
+from openviking.storage.queuefs import get_queue_manager
from openviking.storage.viking_fs import VikingFS, init_viking_fs
from openviking.utils import get_logger
+from openviking.utils.config import OpenVikingConfig, get_openviking_config
+from openviking.utils.config.open_viking_config import initialize_openviking_config
from openviking.utils.config.storage_config import StorageConfig
from openviking.utils.resource_processor import ResourceProcessor
from openviking.utils.skill_processor import SkillProcessor
-
+from openviking.utils.uri import VikingURI
logger = get_logger(__name__)
@@ -597,7 +591,7 @@ class AsyncOpenViking:
if not self._viking_fs:
raise RuntimeError("VikingFS not initialized")
await self._viking_fs.mkdir(uri)
-
+
async def stat(self, uri: str) -> Dict:
"""Get resource status"""
await self._ensure_initialized()
diff --git a/openviking/core/__init__.py b/openviking/core/__init__.py
index e8c575450..19e15538c 100644
--- a/openviking/core/__init__.py
+++ b/openviking/core/__init__.py
@@ -2,13 +2,13 @@
# SPDX-License-Identifier: Apache-2.0
"""Core context abstractions for OpenViking."""
+from openviking.core.building_tree import BuildingTree
from openviking.core.context import Context, ResourceContentType
from openviking.core.directories import (
PRESET_DIRECTORIES,
DirectoryDefinition,
DirectoryInitializer,
)
-from openviking.core.building_tree import BuildingTree
from openviking.core.skill_loader import SkillLoader
__all__ = [
diff --git a/openviking/core/context.py b/openviking/core/context.py
index 12c51ee60..6d37fc187 100644
--- a/openviking/core/context.py
+++ b/openviking/core/context.py
@@ -10,11 +10,13 @@ from uuid import uuid4
class ResourceContentType(str, Enum):
"""Resource content type"""
+
TEXT = "text"
IMAGE = "image"
VIDEO = "video"
AUDIO = "audio"
+
class ContextType(str, Enum):
"""Context type"""
@@ -22,6 +24,7 @@ class ContextType(str, Enum):
MEMORY = "memory"
RESOURCE = "resource"
+
class Vectorize:
text: str = ""
# image: str = ""
@@ -31,6 +34,7 @@ class Vectorize:
def __init__(self, text: str = ""):
self.text = text
+
class Context:
"""
Unified context class for all context types in OpenViking.
@@ -80,7 +84,7 @@ class Context:
return "memory"
else:
return "resource"
-
+
def _derive_category(self) -> str:
"""Derive category from URI prefix."""
if self.uri.startswith("viking://agent/memories"):
@@ -105,7 +109,7 @@ class Context:
def set_vectorize(self, vectorize: Vectorize):
self.vectorize = vectorize
-
+
def get_vectorization_text(self) -> str:
"""Get text for vectorization."""
# todo: multi-modal support
diff --git a/openviking/core/directories.py b/openviking/core/directories.py
index d82e564c9..a7e87e575 100644
--- a/openviking/core/directories.py
+++ b/openviking/core/directories.py
@@ -8,10 +8,11 @@ This module defines the preset directory structure that is created on initializa
"""
from dataclasses import dataclass, field
-from datetime import datetime
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Dict, List, Optional
+
from openviking.core.context import Context, ContextType, Vectorize
from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter
+
if TYPE_CHECKING:
from openviking.storage import VikingDBManager
@@ -198,8 +199,6 @@ class DirectoryInitializer:
count = 1 if created else 0
count += await self._initialize_children("user", user_tree.children, user_root_uri)
- from openviking.utils.logger import get_logger
-
return count
async def _ensure_directory(
@@ -225,11 +224,11 @@ class DirectoryInitializer:
)
if not existing:
context = Context(
- uri=uri,
- parent_uri=parent_uri,
- is_leaf=False,
- context_type=get_context_type_for_uri(uri),
- abstract=defn.abstract,
+ uri=uri,
+ parent_uri=parent_uri,
+ is_leaf=False,
+ context_type=get_context_type_for_uri(uri),
+ abstract=defn.abstract,
)
context.set_vectorize(Vectorize(text=defn.overview))
dir_emb_msg = EmbeddingMsgConverter.from_context(context)
@@ -282,6 +281,7 @@ class DirectoryInitializer:
async def _create_agfs_structure(self, uri: str, abstract: str, overview: str) -> None:
"""Create L0/L1 file structure for directory in AGFS."""
from openviking.storage.viking_fs import get_viking_fs
+
await get_viking_fs().write_context(
uri=uri,
abstract=abstract,
diff --git a/openviking/core/mcp_converter.py b/openviking/core/mcp_converter.py
index 51dae6113..2faabaa9b 100644
--- a/openviking/core/mcp_converter.py
+++ b/openviking/core/mcp_converter.py
@@ -2,7 +2,7 @@
# SPDX-License-Identifier: Apache-2.0
"""MCP to Skill converter."""
-from typing import Dict, Any
+from typing import Any, Dict
def mcp_to_skill(mcp_config: Dict[str, Any]) -> Dict[str, Any]:
@@ -42,7 +42,9 @@ def mcp_to_skill(mcp_config: Dict[str, Any]) -> Dict[str, Any]:
# Add usage section
body_parts.append("\n## Usage\n\n")
body_parts.append(f"This tool wraps the MCP tool `{name}`. ")
- body_parts.append("Call this when the user needs functionality matching the description above.\n")
+ body_parts.append(
+ "Call this when the user needs functionality matching the description above.\n"
+ )
content = "".join(frontmatter_parts) + "".join(body_parts)
diff --git a/openviking/message/part.py b/openviking/message/part.py
index df495e2c0..870debada 100644
--- a/openviking/message/part.py
+++ b/openviking/message/part.py
@@ -6,7 +6,7 @@ Message consists of multiple Parts, each Part has different type and purpose.
"""
from dataclasses import dataclass
-from typing import Literal, Union, Optional
+from typing import Literal, Optional, Union
@dataclass
diff --git a/openviking/models/embedder/__init__.py b/openviking/models/embedder/__init__.py
index dba382f11..30f459590 100644
--- a/openviking/models/embedder/__init__.py
+++ b/openviking/models/embedder/__init__.py
@@ -14,32 +14,25 @@ Supported providers:
"""
from openviking.models.embedder.base import (
- EmbedResult,
- EmbedderBase,
+ CompositeHybridEmbedder,
DenseEmbedderBase,
- SparseEmbedderBase,
+ EmbedderBase,
+ EmbedResult,
HybridEmbedderBase,
- CompositeHybridEmbedder,
+ SparseEmbedderBase,
)
-
-from openviking.models.embedder.openai_embedders import (
- OpenAIDenseEmbedder,
+from openviking.models.embedder.openai_embedders import OpenAIDenseEmbedder
+from openviking.models.embedder.vikingdb_embedders import (
+ VikingDBDenseEmbedder,
+ VikingDBHybridEmbedder,
+ VikingDBSparseEmbedder,
)
-
from openviking.models.embedder.volcengine_embedders import (
VolcengineDenseEmbedder,
- VolcengineSparseEmbedder,
VolcengineHybridEmbedder,
+ VolcengineSparseEmbedder,
)
-from openviking.models.embedder.vikingdb_embedders import (
- VikingDBDenseEmbedder,
- VikingDBSparseEmbedder,
- VikingDBHybridEmbedder,
-)
-
-
-
__all__ = [
# Base classes
"EmbedResult",
@@ -48,15 +41,12 @@ __all__ = [
"SparseEmbedderBase",
"HybridEmbedderBase",
"CompositeHybridEmbedder",
-
# OpenAI implementations
"OpenAIDenseEmbedder",
-
# Volcengine implementations
"VolcengineDenseEmbedder",
"VolcengineSparseEmbedder",
"VolcengineHybridEmbedder",
-
# VikingDB implementations
"VikingDBDenseEmbedder",
"VikingDBSparseEmbedder",
diff --git a/openviking/models/embedder/base.py b/openviking/models/embedder/base.py
index a8cba3bb9..7e7a43690 100644
--- a/openviking/models/embedder/base.py
+++ b/openviking/models/embedder/base.py
@@ -1,8 +1,8 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
from abc import ABC, abstractmethod
-from typing import List, Dict, Any, Optional
from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
def truncate_and_normalize(embedding: List[float], dimension: Optional[int]) -> List[float]:
@@ -19,6 +19,7 @@ def truncate_and_normalize(embedding: List[float], dimension: Optional[int]) ->
return embedding
import math
+
embedding = embedding[:dimension]
norm = math.sqrt(sum(x**2 for x in embedding))
if norm > 0:
@@ -34,6 +35,7 @@ class EmbedResult:
dense_vector: Dense vector in List[float] format
sparse_vector: Sparse vector in Dict[str, float] format, e.g. {'token1': 0.5, 'token2': 0.3}
"""
+
dense_vector: Optional[List[float]] = None
sparse_vector: Optional[Dict[str, float]] = None
@@ -170,7 +172,6 @@ class SparseEmbedderBase(EmbedderBase):
return True
-
class HybridEmbedderBase(EmbedderBase):
"""Hybrid embedder base class that returns both dense and sparse vectors
@@ -225,9 +226,7 @@ class CompositeHybridEmbedder(HybridEmbedderBase):
def __init__(self, dense_embedder: DenseEmbedderBase, sparse_embedder: SparseEmbedderBase):
"""Initialize with two separate embedders"""
- super().__init__(
- model_name=f"{dense_embedder.model_name}+{sparse_embedder.model_name}"
- )
+ super().__init__(model_name=f"{dense_embedder.model_name}+{sparse_embedder.model_name}")
self.dense_embedder = dense_embedder
self.sparse_embedder = sparse_embedder
@@ -235,22 +234,18 @@ class CompositeHybridEmbedder(HybridEmbedderBase):
"""Combine results from both embedders"""
dense_res = self.dense_embedder.embed(text)
sparse_res = self.sparse_embedder.embed(text)
-
+
return EmbedResult(
- dense_vector=dense_res.dense_vector,
- sparse_vector=sparse_res.sparse_vector
+ dense_vector=dense_res.dense_vector, sparse_vector=sparse_res.sparse_vector
)
def embed_batch(self, texts: List[str]) -> List[EmbedResult]:
"""Combine batch results"""
dense_results = self.dense_embedder.embed_batch(texts)
sparse_results = self.sparse_embedder.embed_batch(texts)
-
+
return [
- EmbedResult(
- dense_vector=d.dense_vector,
- sparse_vector=s.sparse_vector
- )
+ EmbedResult(dense_vector=d.dense_vector, sparse_vector=s.sparse_vector)
for d, s in zip(dense_results, sparse_results)
]
@@ -259,4 +254,4 @@ class CompositeHybridEmbedder(HybridEmbedderBase):
def close(self):
self.dense_embedder.close()
- self.sparse_embedder.close()
\ No newline at end of file
+ self.sparse_embedder.close()
diff --git a/openviking/models/embedder/openai_embedders.py b/openviking/models/embedder/openai_embedders.py
index 75d14fd1c..34f3b5933 100644
--- a/openviking/models/embedder/openai_embedders.py
+++ b/openviking/models/embedder/openai_embedders.py
@@ -1,14 +1,17 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
"""OpenAI Embedder Implementation"""
+
import os
-from typing import List, Optional, Dict, Any
+from typing import Any, Dict, List, Optional
+
import openai
+
from openviking.models.embedder.base import (
DenseEmbedderBase,
- SparseEmbedderBase,
+ EmbedResult,
HybridEmbedderBase,
- EmbedResult
+ SparseEmbedderBase,
)
@@ -34,7 +37,7 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):
api_key: Optional[str] = None,
api_base: Optional[str] = None,
dimension: Optional[int] = None,
- config: Optional[Dict[str, Any]] = None
+ config: Optional[Dict[str, Any]] = None,
):
"""Initialize OpenAI Dense Embedder
@@ -50,7 +53,9 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):
"""
super().__init__(model_name, config)
- self.api_key = api_key or os.getenv("OPENVIKING_EMBEDDING_API_KEY") or os.getenv("OPENAI_API_KEY")
+ self.api_key = (
+ api_key or os.getenv("OPENVIKING_EMBEDDING_API_KEY") or os.getenv("OPENAI_API_KEY")
+ )
self.api_base = api_base or os.getenv("OPENVIKING_EMBEDDING_API_BASE")
self.dimension = dimension
@@ -128,10 +133,7 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):
response = self.client.embeddings.create(**kwargs)
- return [
- EmbedResult(dense_vector=item.embedding)
- for item in response.data
- ]
+ return [EmbedResult(dense_vector=item.embedding) for item in response.data]
except openai.APIError as e:
raise RuntimeError(f"OpenAI API error: {e.message}") from e
except Exception as e:
diff --git a/openviking/models/embedder/vikingdb_embedders.py b/openviking/models/embedder/vikingdb_embedders.py
index 3f99a69e4..8ba507b96 100644
--- a/openviking/models/embedder/vikingdb_embedders.py
+++ b/openviking/models/embedder/vikingdb_embedders.py
@@ -1,20 +1,23 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
"""VikingDB Embedder Implementation via HTTP API"""
+
import os
-from typing import List, Optional, Dict, Any
+from typing import Any, Dict, List, Optional
+
from openviking.models.embedder.base import (
DenseEmbedderBase,
- SparseEmbedderBase,
+ EmbedResult,
HybridEmbedderBase,
- EmbedResult
+ SparseEmbedderBase,
)
from openviking.storage.vectordb.collection.volcengine_clients import ClientForDataApi
from openviking.utils.logger import default_logger as logger
+
class VikingDBClientMixin:
"""Mixin to handle VikingDB Client initialization and API calls."""
-
+
def _init_vikingdb_client(
self,
ak: Optional[str] = None,
@@ -32,15 +35,18 @@ class VikingDBClientMixin:
self.client = ClientForDataApi(self.ak, self.sk, self.region, self.host)
- def _call_api(self, texts: List[str], dense_model: Dict[str, Any] = None, sparse_model: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]:
+ def _call_api(
+ self,
+ texts: List[str],
+ dense_model: Dict[str, Any] = None,
+ sparse_model: Optional[Dict[str, Any]] = None,
+ ) -> List[Dict[str, Any]]:
"""Call VikingDB Embedding API"""
path = "/api/vikingdb/embedding"
-
+
data_items = [{"text": text} for text in texts]
-
- req_body = {
- "data": data_items
- }
+
+ req_body = {"data": data_items}
if dense_model:
req_body["dense_model"] = dense_model
if sparse_model:
@@ -49,22 +55,27 @@ class VikingDBClientMixin:
try:
response = self.client.do_req("POST", path, req_body=req_body)
if response.status_code != 200:
- logger.warning(f"VikingDB API returned bad code: {response.status_code}, message: {response.text}")
+ logger.warning(
+ f"VikingDB API returned bad code: {response.status_code}, message: {response.text}"
+ )
return []
-
+
result = response.json()
return result.get("result", {}).get("data", [])
-
+
except Exception as e:
logger.error(f"Failed to get embeddings: {e}")
raise e
- def _truncate_and_normalize(self, embedding: List[float], dimension: Optional[int]) -> List[float]:
+ def _truncate_and_normalize(
+ self, embedding: List[float], dimension: Optional[int]
+ ) -> List[float]:
"""Truncate and L2 normalize embedding"""
if not dimension or len(embedding) <= dimension:
return embedding
import math
+
embedding = embedding[:dimension]
norm = math.sqrt(sum(x**2 for x in embedding))
if norm > 0:
@@ -79,7 +90,7 @@ class VikingDBClientMixin:
result = {}
if isinstance(sparse_data, dict):
return {str(k): float(v) for k, v in sparse_data.items()}
-
+
if isinstance(sparse_data, list):
for item in sparse_data:
if isinstance(item, dict):
@@ -93,7 +104,7 @@ class VikingDBClientMixin:
class VikingDBDenseEmbedder(DenseEmbedderBase, VikingDBClientMixin):
"""VikingDB Dense Embedder"""
-
+
def __init__(
self,
model_name: str,
@@ -104,29 +115,25 @@ class VikingDBDenseEmbedder(DenseEmbedderBase, VikingDBClientMixin):
host: Optional[str] = None,
dimension: Optional[int] = None,
embedding_type: str = "text",
- config: Optional[Dict[str, Any]] = None
+ config: Optional[Dict[str, Any]] = None,
):
DenseEmbedderBase.__init__(self, model_name, config)
self._init_vikingdb_client(ak, sk, region, host)
self.model_version = model_version
self.dimension = dimension
self.embedding_type = embedding_type
- self.dense_model = {
- "name": model_name,
- "version": model_version,
- "dim": dimension
- }
+ self.dense_model = {"name": model_name, "version": model_version, "dim": dimension}
def embed(self, text: str) -> EmbedResult:
results = self._call_api([text], dense_model=self.dense_model)
if not results:
return EmbedResult(dense_vector=[])
-
+
item = results[0]
dense_vector = []
if "dense_embedding" in item:
dense_vector = self._truncate_and_normalize(item["dense_embedding"], self.dimension)
-
+
return EmbedResult(dense_vector=dense_vector)
def embed_batch(self, texts: List[str]) -> List[EmbedResult]:
@@ -134,17 +141,21 @@ class VikingDBDenseEmbedder(DenseEmbedderBase, VikingDBClientMixin):
return []
raw_results = self._call_api(texts, dense_model=self.dense_model)
return [
- EmbedResult(dense_vector=self._truncate_and_normalize(item.get("dense_embedding", []), self.dimension))
+ EmbedResult(
+ dense_vector=self._truncate_and_normalize(
+ item.get("dense_embedding", []), self.dimension
+ )
+ )
for item in raw_results
]
-
+
def get_dimension(self) -> int:
return self.dimension if self.dimension else 2048
class VikingDBSparseEmbedder(SparseEmbedderBase, VikingDBClientMixin):
"""VikingDB Sparse Embedder"""
-
+
def __init__(
self,
model_name: str,
@@ -153,7 +164,7 @@ class VikingDBSparseEmbedder(SparseEmbedderBase, VikingDBClientMixin):
sk: Optional[str] = None,
region: Optional[str] = None,
host: Optional[str] = None,
- config: Optional[Dict[str, Any]] = None
+ config: Optional[Dict[str, Any]] = None,
):
SparseEmbedderBase.__init__(self, model_name, config)
self._init_vikingdb_client(ak, sk, region, host)
@@ -167,12 +178,12 @@ class VikingDBSparseEmbedder(SparseEmbedderBase, VikingDBClientMixin):
results = self._call_api([text], sparse_model=self.sparse_model)
if not results:
return EmbedResult(sparse_vector={})
-
+
item = results[0]
sparse_vector = {}
if "sparse" in item:
sparse_vector = item["sparse"]
-
+
return EmbedResult(sparse_vector=sparse_vector)
def embed_batch(self, texts: List[str]) -> List[EmbedResult]:
@@ -180,14 +191,16 @@ class VikingDBSparseEmbedder(SparseEmbedderBase, VikingDBClientMixin):
return []
raw_results = self._call_api(texts, sparse_model=self.sparse_model)
return [
- EmbedResult(sparse_vector=self._process_sparse_embedding(item.get("sparse_embedding", {})))
+ EmbedResult(
+ sparse_vector=self._process_sparse_embedding(item.get("sparse_embedding", {}))
+ )
for item in raw_results
]
class VikingDBHybridEmbedder(HybridEmbedderBase, VikingDBClientMixin):
"""VikingDB Hybrid Embedder"""
-
+
def __init__(
self,
model_name: str,
@@ -198,29 +211,26 @@ class VikingDBHybridEmbedder(HybridEmbedderBase, VikingDBClientMixin):
host: Optional[str] = None,
dimension: Optional[int] = None,
embedding_type: str = "text",
- config: Optional[Dict[str, Any]] = None
+ config: Optional[Dict[str, Any]] = None,
):
HybridEmbedderBase.__init__(self, model_name, config)
self._init_vikingdb_client(ak, sk, region, host)
self.model_version = model_version
self.dimension = dimension
self.embedding_type = embedding_type
- self.dense_model = {
- "name": model_name,
- "version": model_version,
- "dim": dimension
- }
+ self.dense_model = {"name": model_name, "version": model_version, "dim": dimension}
self.sparse_model = {
"name": model_name,
"version": model_version,
}
-
def embed(self, text: str) -> EmbedResult:
- results = self._call_api([text], dense_model=self.dense_model, sparse_model=self.sparse_model)
+ results = self._call_api(
+ [text], dense_model=self.dense_model, sparse_model=self.sparse_model
+ )
if not results:
return EmbedResult(dense_vector=[], sparse_vector={})
-
+
item = results[0]
dense_vector = []
sparse_vector = {}
@@ -229,13 +239,15 @@ class VikingDBHybridEmbedder(HybridEmbedderBase, VikingDBClientMixin):
dense_vector = self._truncate_and_normalize(item["dense"], self.dimension)
if "sparse" in item:
sparse_vector = item["sparse"]
-
+
return EmbedResult(dense_vector=dense_vector, sparse_vector=sparse_vector)
def embed_batch(self, texts: List[str]) -> List[EmbedResult]:
if not texts:
return []
- raw_results = self._call_api(texts, dense_model=self.dense_model, sparse_model=self.sparse_model)
+ raw_results = self._call_api(
+ texts, dense_model=self.dense_model, sparse_model=self.sparse_model
+ )
results = []
for item in raw_results:
if "dense" in item:
diff --git a/openviking/models/embedder/volcengine_embedders.py b/openviking/models/embedder/volcengine_embedders.py
index 6cd764213..5c3ca0d28 100644
--- a/openviking/models/embedder/volcengine_embedders.py
+++ b/openviking/models/embedder/volcengine_embedders.py
@@ -1,15 +1,18 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
"""Volcengine Embedder Implementation"""
+
import os
-from typing import List, Optional, Dict, Any, Union
+from typing import Any, Dict, List, Optional
+
import volcenginesdkarkruntime
+
from openviking.models.embedder.base import (
DenseEmbedderBase,
- SparseEmbedderBase,
- HybridEmbedderBase,
EmbedResult,
- truncate_and_normalize
+ HybridEmbedderBase,
+ SparseEmbedderBase,
+ truncate_and_normalize,
)
@@ -18,17 +21,17 @@ def process_sparse_embedding(sparse_data: Any) -> Dict[str, float]:
if not sparse_data:
return {}
result = {}
-
+
# Helper to extract index/value from an item (dict or object)
def extract_pair(item):
- idx = getattr(item, 'index', None)
+ idx = getattr(item, "index", None)
if idx is None and isinstance(item, dict):
- idx = item.get('index')
-
- val = getattr(item, 'value', None)
+ idx = item.get("index")
+
+ val = getattr(item, "value", None)
if val is None and isinstance(item, dict):
- val = item.get('value')
-
+ val = item.get("value")
+
return idx, val
if isinstance(sparse_data, list):
@@ -36,14 +39,14 @@ def process_sparse_embedding(sparse_data: Any) -> Dict[str, float]:
idx, val = extract_pair(item)
if idx is not None and val is not None:
result[str(idx)] = float(val)
- elif hasattr(sparse_data, 'index'):
- # Single object case (unlikely for vector but possible per type hint)
- idx, val = extract_pair(sparse_data)
- if idx is not None and val is not None:
- result[str(idx)] = float(val)
+ elif hasattr(sparse_data, "index"):
+ # Single object case (unlikely for vector but possible per type hint)
+ idx, val = extract_pair(sparse_data)
+ if idx is not None and val is not None:
+ result[str(idx)] = float(val)
elif isinstance(sparse_data, dict):
# Maybe a direct dict?
- return {str(k): float(v) for k, v in sparse_data.items()}
+ return {str(k): float(v) for k, v in sparse_data.items()}
return result
@@ -61,7 +64,7 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):
api_base: Optional[str] = None,
dimension: Optional[int] = None,
input_type: str = "multimodal",
- config: Optional[Dict[str, Any]] = None
+ config: Optional[Dict[str, Any]] = None,
):
"""Initialize Volcengine Dense Embedder
@@ -78,8 +81,14 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):
"""
super().__init__(model_name, config)
- self.api_key = api_key or os.getenv("OPENVIKING_EMBEDDING_API_KEY") or os.getenv("VOLC_API_KEY")
- self.api_base = api_base or os.getenv("OPENVIKING_EMBEDDING_API_BASE") or "https://ark.cn-beijing.volces.com/api/v3"
+ self.api_key = (
+ api_key or os.getenv("OPENVIKING_EMBEDDING_API_KEY") or os.getenv("VOLC_API_KEY")
+ )
+ self.api_base = (
+ api_base
+ or os.getenv("OPENVIKING_EMBEDDING_API_BASE")
+ or "https://ark.cn-beijing.volces.com/api/v3"
+ )
self.dimension = dimension
self.input_type = input_type
@@ -87,10 +96,7 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):
raise ValueError("api_key is required")
# Initialize Volcengine client
- self.client = volcenginesdkarkruntime.Ark(
- api_key=self.api_key,
- base_url=self.api_base
- )
+ self.client = volcenginesdkarkruntime.Ark(api_key=self.api_key, base_url=self.api_base)
# Auto-detect dimension
self._dimension = dimension
@@ -121,16 +127,12 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):
if self.input_type == "multimodal":
# Use multimodal embeddings API
response = self.client.multimodal_embeddings.create(
- input=[{'type': 'text', 'text': text}],
- model=self.model_name
+ input=[{"type": "text", "text": text}], model=self.model_name
)
vector = response.data.embedding
else:
# Use text embeddings API
- response = self.client.embeddings.create(
- input=text,
- model=self.model_name
- )
+ response = self.client.embeddings.create(input=text, model=self.model_name)
vector = response.data[0].embedding
vector = truncate_and_normalize(vector, self.dimension)
@@ -155,17 +157,13 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):
try:
if self.input_type == "multimodal":
- multimodal_inputs = [{'type': 'text', 'text': text} for text in texts]
+ multimodal_inputs = [{"type": "text", "text": text} for text in texts]
response = self.client.multimodal_embeddings.create(
- input=multimodal_inputs,
- model=self.model_name
+ input=multimodal_inputs, model=self.model_name
)
data = response.data
else:
- response = self.client.embeddings.create(
- input=texts,
- model=self.model_name
- )
+ response = self.client.embeddings.create(input=texts, model=self.model_name)
data = response.data
return [
@@ -190,7 +188,7 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):
model_name: str,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
- config: Optional[Dict[str, Any]] = None
+ config: Optional[Dict[str, Any]] = None,
):
"""Initialize Volcengine Sparse Embedder
@@ -206,17 +204,12 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):
super().__init__(model_name, config)
self.api_key = api_key
- self.api_base = api_base
-
+ self.api_base = api_base
if not self.api_key:
raise ValueError("api_key is required")
- self.client = volcenginesdkarkruntime.Ark(
- api_key=self.api_key,
- base_url=self.api_base
- )
-
+ self.client = volcenginesdkarkruntime.Ark(api_key=self.api_key, base_url=self.api_base)
def embed(self, text: str) -> EmbedResult:
"""Perform sparse embedding on text
@@ -233,12 +226,12 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):
try:
# Must use multimodal endpoint for sparse
response = self.client.multimodal_embeddings.create(
- input=[{'type': 'text', 'text': text}],
+ input=[{"type": "text", "text": text}],
model=self.model_name,
- sparse_embedding={'type': 'enabled'}
+ sparse_embedding={"type": "enabled"},
)
item = response.data[0]
- sparse_vector = getattr(item, 'sparse_embedding', None)
+ sparse_vector = getattr(item, "sparse_embedding", None)
return EmbedResult(sparse_vector=process_sparse_embedding(sparse_vector))
except Exception as e:
raise RuntimeError(f"Volcengine sparse embedding failed: {str(e)}") from e
@@ -255,17 +248,16 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):
Raises:
RuntimeError: When API call fails
"""
- if not texts: return []
+ if not texts:
+ return []
try:
- multimodal_inputs = [{'type': 'text', 'text': text} for text in texts]
+ multimodal_inputs = [{"type": "text", "text": text} for text in texts]
response = self.client.multimodal_embeddings.create(
- input=multimodal_inputs,
- model=self.model_name,
- sparse_embedding={'type': 'enabled'}
+ input=multimodal_inputs, model=self.model_name, sparse_embedding={"type": "enabled"}
)
results = []
for item in response.data:
- sparse_vector = getattr(item, 'sparse_embedding', None)
+ sparse_vector = getattr(item, "sparse_embedding", None)
results.append(EmbedResult(sparse_vector=process_sparse_embedding(sparse_vector)))
return results
except Exception as e:
@@ -286,7 +278,7 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):
api_base: Optional[str] = None,
dimension: Optional[int] = None,
input_type: str = "multimodal",
- config: Optional[Dict[str, Any]] = None
+ config: Optional[Dict[str, Any]] = None,
):
"""Initialize Volcengine Hybrid Embedder
@@ -310,10 +302,7 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):
if not self.api_key:
raise ValueError("api_key is required")
- self.client = volcenginesdkarkruntime.Ark(
- api_key=self.api_key,
- base_url=self.api_base
- )
+ self.client = volcenginesdkarkruntime.Ark(api_key=self.api_key, base_url=self.api_base)
self._dimension = dimension or 2048
def embed(self, text: str) -> EmbedResult:
@@ -332,17 +321,16 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):
# Always use multimodal for hybrid to get both
response = self.client.multimodal_embeddings.create(
- input=[{'type': 'text', 'text': text}],
+ input=[{"type": "text", "text": text}],
model=self.model_name,
- sparse_embedding={'type': 'enabled'}
+ sparse_embedding={"type": "enabled"},
)
item = response.data
dense_vector = truncate_and_normalize(item.embedding, self.dimension)
- sparse_vector = getattr(item, 'sparse_embedding', None)
+ sparse_vector = getattr(item, "sparse_embedding", None)
return EmbedResult(
- dense_vector=dense_vector,
- sparse_vector=process_sparse_embedding(sparse_vector)
+ dense_vector=dense_vector, sparse_vector=process_sparse_embedding(sparse_vector)
)
except Exception as e:
raise RuntimeError(f"Volcengine hybrid embedding failed: {str(e)}") from e
@@ -359,22 +347,23 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):
Raises:
RuntimeError: When API call fails
"""
- if not texts: return []
+ if not texts:
+ return []
try:
- multimodal_inputs = [{'type': 'text', 'text': text} for text in texts]
+ multimodal_inputs = [{"type": "text", "text": text} for text in texts]
response = self.client.multimodal_embeddings.create(
- input=multimodal_inputs,
- model=self.model_name,
- sparse_embedding={'type': 'enabled'}
+ input=multimodal_inputs, model=self.model_name, sparse_embedding={"type": "enabled"}
)
results = []
for item in response.data:
dense_vector = truncate_and_normalize(item.embedding, self.dimension)
- sparse_vector = getattr(item, 'sparse_embedding', None)
- results.append(EmbedResult(
- dense_vector=dense_vector,
- sparse_vector=process_sparse_embedding(sparse_vector)
- ))
+ sparse_vector = getattr(item, "sparse_embedding", None)
+ results.append(
+ EmbedResult(
+ dense_vector=dense_vector,
+ sparse_vector=process_sparse_embedding(sparse_vector),
+ )
+ )
return results
except Exception as e:
raise RuntimeError(f"Volcengine batch hybrid embedding failed: {str(e)}") from e
diff --git a/openviking/models/vlm/__init__.py b/openviking/models/vlm/__init__.py
index 598d126ec..8e38aceae 100644
--- a/openviking/models/vlm/__init__.py
+++ b/openviking/models/vlm/__init__.py
@@ -2,9 +2,9 @@
# SPDX-License-Identifier: Apache-2.0
"""VLM (Vision-Language Model) module"""
-from .base import VLMBase, VLMFactory
from .backends.openai_vlm import OpenAIVLM
from .backends.volcengine_vlm import VolcEngineVLM
+from .base import VLMBase, VLMFactory
__all__ = [
"VLMBase",
diff --git a/openviking/models/vlm/backends/openai_vlm.py b/openviking/models/vlm/backends/openai_vlm.py
index 14f8b1939..128da40bb 100644
--- a/openviking/models/vlm/backends/openai_vlm.py
+++ b/openviking/models/vlm/backends/openai_vlm.py
@@ -2,10 +2,10 @@
# SPDX-License-Identifier: Apache-2.0
"""OpenAI VLM backend implementation"""
-import base64
import asyncio
+import base64
from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Union
from ..base import VLMBase
@@ -63,12 +63,14 @@ class OpenAIVLM(VLMBase):
if self.provider == "volcengine":
kwargs["thinking"] = {"type": "disabled" if not thinking else "enabled"}
-
+
response = client.chat.completions.create(**kwargs)
self._update_token_usage_from_response(response)
return response.choices[0].message.content or ""
- async def get_completion_async(self, prompt: str, thinking: bool = False, max_retries: int = 0) -> str:
+ async def get_completion_async(
+ self, prompt: str, thinking: bool = False, max_retries: int = 0
+ ) -> str:
"""Get text completion asynchronously"""
client = self.get_async_client()
kwargs = {
diff --git a/openviking/models/vlm/backends/volcengine_vlm.py b/openviking/models/vlm/backends/volcengine_vlm.py
index 8f1d2e97c..f11a289e6 100644
--- a/openviking/models/vlm/backends/volcengine_vlm.py
+++ b/openviking/models/vlm/backends/volcengine_vlm.py
@@ -2,13 +2,12 @@
# SPDX-License-Identifier: Apache-2.0
"""VolcEngine VLM backend implementation"""
-import base64
from pathlib import Path
-from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING
+from typing import Any, Dict, List, Union
-from ..base import VLMBase
from .openai_vlm import OpenAIVLM
+
class VolcEngineVLM(OpenAIVLM):
"""VolcEngine VLM backend"""
@@ -74,4 +73,3 @@ class VolcEngineVLM(OpenAIVLM):
images: List[Union[str, Path, bytes]],
) -> str:
return await super().get_vision_completion_async(prompt, images)
-
diff --git a/openviking/models/vlm/base.py b/openviking/models/vlm/base.py
index 80dfacc13..25d63aa01 100644
--- a/openviking/models/vlm/base.py
+++ b/openviking/models/vlm/base.py
@@ -4,7 +4,7 @@
from abc import ABC, abstractmethod
from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Union
from .token_usage import TokenUsageTracker
@@ -120,12 +120,14 @@ class VLMFactory:
ImportError: If related dependencies are not installed
"""
provider = config.get("provider") or config.get("backend") or "openai"
-
+
if provider == "openai":
from .backends.openai_vlm import OpenAIVLM
+
return OpenAIVLM(config)
elif provider == "volcengine":
from .backends.volcengine_vlm import VolcEngineVLM
+
return VolcEngineVLM(config)
else:
raise ValueError(f"Unsupported VLM provider: {provider}")
diff --git a/openviking/models/vlm/llm.py b/openviking/models/vlm/llm.py
index c59c2a8b6..f657f0304 100644
--- a/openviking/models/vlm/llm.py
+++ b/openviking/models/vlm/llm.py
@@ -10,10 +10,10 @@ import json
import re
from typing import Any, Dict, Optional, Type, TypeVar
+import json_repair
from pydantic import BaseModel
from openviking.utils.logger import get_logger
-import json_repair
logger = get_logger(__name__)
diff --git a/openviking/models/vlm/token_usage.py b/openviking/models/vlm/token_usage.py
index f1b511946..3df8db6ab 100644
--- a/openviking/models/vlm/token_usage.py
+++ b/openviking/models/vlm/token_usage.py
@@ -111,7 +111,10 @@ class ModelTokenUsage:
def __str__(self) -> str:
providers = ", ".join(
- [f"{provider}: {usage.total_tokens}" for provider, usage in self.usage_by_provider.items()]
+ [
+ f"{provider}: {usage.total_tokens}"
+ for provider, usage in self.usage_by_provider.items()
+ ]
)
return f"ModelTokenUsage(model={self.model_name}, total={self.total_usage.total_tokens}, providers=[{providers}])"
diff --git a/openviking/parse/__init__.py b/openviking/parse/__init__.py
index 0a8414d21..c6dff700c 100644
--- a/openviking/parse/__init__.py
+++ b/openviking/parse/__init__.py
@@ -2,18 +2,9 @@
# SPDX-License-Identifier: Apache-2.0
"""Document parsers for various formats."""
-from openviking.parse.base import (
- NodeType,
- ParseResult,
- ResourceNode,
- create_parse_result,
-)
+from openviking.parse.base import NodeType, ParseResult, ResourceNode, create_parse_result
from openviking.parse.converter import DocumentConverter
-from openviking.parse.custom import (
- CallbackParserWrapper,
- CustomParserProtocol,
- CustomParserWrapper,
-)
+from openviking.parse.custom import CallbackParserWrapper, CustomParserProtocol, CustomParserWrapper
from openviking.parse.parsers.base_parser import BaseParser
from openviking.parse.parsers.code import CodeParser
from openviking.parse.parsers.html import HTMLParser
diff --git a/openviking/parse/base.py b/openviking/parse/base.py
index 71de87976..9548f4b4c 100644
--- a/openviking/parse/base.py
+++ b/openviking/parse/base.py
@@ -7,15 +7,14 @@ Following PageIndex philosophy: preserve natural document structure
rather than arbitrary chunking.
"""
-from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from datetime import datetime
from enum import Enum
from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
if TYPE_CHECKING:
- from openviking.storage.viking_fs import VikingFS
+ pass
# ============================================================================
# Common utility functions
@@ -97,7 +96,9 @@ def lazy_import(module_name: str, package_name: Optional[str] = None) -> Any:
return importlib.import_module(module_name)
except ImportError:
pkg = package_name or module_name
- raise ImportError(f"Module '{module_name}' not available. Please install: pip install {pkg}")
+ raise ImportError(
+ f"Module '{module_name}' not available. Please install: pip install {pkg}"
+ )
class ResourceCategory(Enum):
@@ -154,6 +155,7 @@ class NodeType(Enum):
ROOT = "root"
SECTION = "section"
+
@dataclass
class ResourceNode:
"""
@@ -200,6 +202,7 @@ class ResourceNode:
async def get_detail_content_async(self, temp_uri: str) -> str:
"""Read detail file content from VikingFS temp directory."""
from openviking.storage.viking_fs import get_viking_fs
+
if not self.detail_file:
return ""
file_uri = f"{temp_uri}/{self.detail_file}"
diff --git a/openviking/parse/custom.py b/openviking/parse/custom.py
index 951569c7e..78636a9fc 100644
--- a/openviking/parse/custom.py
+++ b/openviking/parse/custom.py
@@ -131,8 +131,7 @@ class CustomParserWrapper:
"""
if not self.custom_parser.can_handle(source):
raise ValueError(
- f"Parser {type(self.custom_parser).__name__} "
- f"cannot handle source: {source}"
+ f"Parser {type(self.custom_parser).__name__} cannot handle source: {source}"
)
return await self.custom_parser.parse(source, **kwargs)
diff --git a/openviking/parse/parsers/base_parser.py b/openviking/parse/parsers/base_parser.py
index 89d5766b1..4345d6232 100644
--- a/openviking/parse/parsers/base_parser.py
+++ b/openviking/parse/parsers/base_parser.py
@@ -3,6 +3,7 @@
from abc import ABC, abstractmethod
from pathlib import Path
from typing import List, Optional, Union
+
from openviking.parse.base import ParseResult
diff --git a/openviking/parse/parsers/code.py b/openviking/parse/parsers/code.py
index 65c16b97f..777ee9173 100644
--- a/openviking/parse/parsers/code.py
+++ b/openviking/parse/parsers/code.py
@@ -7,10 +7,10 @@ Supports multiple programming languages with syntax-aware parsing.
"""
from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
+from typing import List, Optional, Union
+from openviking.parse.base import ParseResult
from openviking.parse.parsers.base_parser import BaseParser
-from openviking.parse.base import NodeType, ParseResult, ResourceNode
from openviking.utils.config.parser_config import CodeConfig
from openviking.utils.logger import get_logger
@@ -164,4 +164,4 @@ class CodeParser(BaseParser):
Raises:
ValueError: If content is empty
"""
- pass
\ No newline at end of file
+ pass
diff --git a/openviking/parse/parsers/html.py b/openviking/parse/parsers/html.py
index 4549c288b..36f1e68ef 100644
--- a/openviking/parse/parsers/html.py
+++ b/openviking/parse/parsers/html.py
@@ -18,7 +18,7 @@ from enum import Enum
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union
from urllib.parse import urlparse
-from openviking.parse.parsers.base_parser import BaseParser
+
from openviking.parse.base import (
NodeType,
ParseResult,
@@ -26,6 +26,7 @@ from openviking.parse.base import (
create_parse_result,
lazy_import,
)
+from openviking.parse.parsers.base_parser import BaseParser
class URLType(Enum):
@@ -68,9 +69,7 @@ class URLTypeDetector:
"application/xhtml+xml": URLType.WEBPAGE,
}
- async def detect(
- self, url: str, timeout: float = 10.0
- ) -> Tuple[URLType, Dict[str, Any]]:
+ async def detect(self, url: str, timeout: float = 10.0) -> Tuple[URLType, Dict[str, Any]]:
"""
Detect URL content type.
@@ -157,6 +156,7 @@ class HTMLParser(BaseParser):
if not hasattr(self, "_readabilipy") or self._readabilipy is None:
try:
from readabilipy import simple_json
+
self._readabilipy = simple_json
except ImportError:
raise ImportError(
@@ -170,6 +170,7 @@ class HTMLParser(BaseParser):
if not hasattr(self, "_markdownify") or self._markdownify is None:
try:
import markdownify
+
self._markdownify = markdownify
except ImportError:
raise ImportError(
@@ -400,9 +401,7 @@ class HTMLParser(BaseParser):
"""
httpx = lazy_import("httpx")
- async with httpx.AsyncClient(
- timeout=self.timeout, follow_redirects=True
- ) as client:
+ async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
headers = {"User-Agent": self.user_agent}
response = await client.get(url, headers=headers)
response.raise_for_status()
@@ -433,9 +432,7 @@ class HTMLParser(BaseParser):
temp_file.close()
# Download
- async with httpx.AsyncClient(
- timeout=self.timeout, follow_redirects=True
- ) as client:
+ async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:
headers = {"User-Agent": self.user_agent}
response = await client.get(url, headers=headers)
response.raise_for_status()
diff --git a/openviking/parse/parsers/markdown.py b/openviking/parse/parsers/markdown.py
index 53f7666dd..6620130ea 100644
--- a/openviking/parse/parsers/markdown.py
+++ b/openviking/parse/parsers/markdown.py
@@ -17,21 +17,13 @@ The parser handles scenarios:
5. Oversized sections without subsections β†’ split by paragraphs
"""
-import json
import re
import time
-import uuid
from pathlib import Path
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
-from openviking.utils.config import get_openviking_config
+from openviking.parse.base import NodeType, ParseResult, ResourceNode, create_parse_result
from openviking.parse.parsers.base_parser import BaseParser
-from openviking.parse.base import (
- NodeType,
- ParseResult,
- ResourceNode,
- create_parse_result,
-)
from openviking.utils.config.parser_config import ParserConfig
from openviking.utils.logger import get_logger
@@ -382,9 +374,7 @@ class MarkdownParser(BaseParser):
await viking_fs.mkdir(root_dir)
# Get document name
- doc_name = self._sanitize_for_path(
- Path(source_path).stem if source_path else "content"
- )
+ doc_name = self._sanitize_for_path(Path(source_path).stem if source_path else "content")
# Small document: save as single file
if estimated_tokens <= max_size:
@@ -410,22 +400,26 @@ class MarkdownParser(BaseParser):
pre_content = content[:first_heading_start].strip()
if pre_content:
pre_tokens = self._estimate_token_count(pre_content)
- sections.append({
- "name": doc_name,
- "content": pre_content,
- "tokens": pre_tokens,
- "has_children": False,
- "heading_idx": None,
- })
+ sections.append(
+ {
+ "name": doc_name,
+ "content": pre_content,
+ "tokens": pre_tokens,
+ "has_children": False,
+ "heading_idx": None,
+ }
+ )
# Add real sections (top-level only for this pass)
min_level = min(h[3] for h in headings)
i = 0
while i < len(headings):
if headings[i][3] == min_level:
- sections.append({
- "heading_idx": i,
- })
+ sections.append(
+ {
+ "heading_idx": i,
+ }
+ )
i += 1
# Process sections with merge logic
@@ -448,7 +442,8 @@ class MarkdownParser(BaseParser):
# Expand section info
expanded = [
- section if section.get("heading_idx") is None
+ section
+ if section.get("heading_idx") is None
else self._get_section_info(content, headings, section["heading_idx"])
for section in sections
]
@@ -481,10 +476,7 @@ class MarkdownParser(BaseParser):
def _can_merge(self, pending: List, tokens: int, max_size: int, has_children: bool) -> bool:
"""Check if section can merge with pending."""
- return (
- sum(t for _, _, t in pending) + tokens <= max_size
- and not has_children
- )
+ return sum(t for _, _, t in pending) + tokens <= max_size and not has_children
async def _try_add_to_pending(
self, viking_fs, parent_dir: str, pending: List, item: Tuple, max_size: int
@@ -547,13 +539,15 @@ class MarkdownParser(BaseParser):
"""Build and process child sections."""
children = []
if section.get("direct_content"):
- children.append({
- "name": name,
- "content": section["direct_content"],
- "tokens": self._estimate_token_count(section["direct_content"]),
- "has_children": False,
- "heading_idx": None,
- })
+ children.append(
+ {
+ "name": name,
+ "content": section["direct_content"],
+ "tokens": self._estimate_token_count(section["direct_content"]),
+ "has_children": False,
+ "heading_idx": None,
+ }
+ )
for child_idx in section.get("child_indices", []):
children.append({"heading_idx": child_idx})
@@ -650,4 +644,3 @@ class MarkdownParser(BaseParser):
cjk_chars = len(re.findall(r"[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]", content))
other_chars = len(re.findall(r"[^\s]", content)) - cjk_chars
return int(cjk_chars * 0.7 + other_chars * 0.3)
-
diff --git a/openviking/parse/parsers/media.py b/openviking/parse/parsers/media.py
index 8a38307ab..9e2e5dbca 100644
--- a/openviking/parse/parsers/media.py
+++ b/openviking/parse/parsers/media.py
@@ -10,17 +10,14 @@ They serve as a design reference for future media parsing capabilities.
For current document parsing (PDF, Markdown, HTML, Text), see other parser modules.
"""
-from PIL import Image
from pathlib import Path
from typing import List, Optional, Union
-from openviking.parse.parsers.base_parser import BaseParser
-from openviking.parse.base import NodeType, ParseResult, ResourceNode
-from openviking.utils.config.parser_config import (
- AudioConfig,
- ImageConfig,
- VideoConfig,
-)
+from PIL import Image
+
+from openviking.parse.base import NodeType, ParseResult, ResourceNode
+from openviking.parse.parsers.base_parser import BaseParser
+from openviking.utils.config.parser_config import AudioConfig, ImageConfig, VideoConfig
# =============================================================================
# Configuration Classes
@@ -117,7 +114,7 @@ class ImageParser(BaseParser):
img = Image.open(file_path)
width, height = img.size
format_str = img.format or ext[1:].upper()
- except Exception as e:
+ except Exception:
width, height = 0, 0
format_str = ext[1:].upper()
@@ -183,7 +180,7 @@ class ImageParser(BaseParser):
TODO: Integrate with actual VLM API (OpenAI GPT-4V, Claude Vision, etc.)
"""
# Fallback implementation - returns basic placeholder
- return f"Image description (VLM integration pending)\n\nThis is an image. VLM description feature has not yet integrated external API."
+ return "Image description (VLM integration pending)\n\nThis is an image. VLM description feature has not yet integrated external API."
async def _ocr_extract(self, image_bytes: bytes, lang: str) -> Optional[str]:
"""
diff --git a/openviking/parse/parsers/pdf.py b/openviking/parse/parsers/pdf.py
index 1c6397a05..7ada2c784 100644
--- a/openviking/parse/parsers/pdf.py
+++ b/openviking/parse/parsers/pdf.py
@@ -16,7 +16,7 @@ import logging
import time
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
-from openviking.parse.parsers.base_parser import BaseParser
+
from openviking.parse.base import (
NodeType,
ParseResult,
@@ -24,6 +24,7 @@ from openviking.parse.base import (
create_parse_result,
lazy_import,
)
+from openviking.parse.parsers.base_parser import BaseParser
from openviking.utils.config.parser_config import PDFConfig
logger = logging.getLogger(__name__)
diff --git a/openviking/parse/parsers/text.py b/openviking/parse/parsers/text.py
index ef559710e..902614315 100644
--- a/openviking/parse/parsers/text.py
+++ b/openviking/parse/parsers/text.py
@@ -8,8 +8,9 @@ Delegates to MarkdownParser since plain text is just unformatted markdown.
from pathlib import Path
from typing import List, Optional, Union
-from openviking.parse.parsers.base_parser import BaseParser
+
from openviking.parse.base import ParseResult
+from openviking.parse.parsers.base_parser import BaseParser
from openviking.utils.config.parser_config import ParserConfig
diff --git a/openviking/parse/registry.py b/openviking/parse/registry.py
index c9acafbbe..023fb1ad0 100644
--- a/openviking/parse/registry.py
+++ b/openviking/parse/registry.py
@@ -8,15 +8,15 @@ Provides automatic parser selection based on file type.
import logging
from pathlib import Path
-from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Type, Union
+from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Union
from openviking.parse.base import ParseResult
from openviking.parse.parsers.base_parser import BaseParser
+from openviking.parse.parsers.code import CodeParser
from openviking.parse.parsers.html import HTMLParser
from openviking.parse.parsers.markdown import MarkdownParser
from openviking.parse.parsers.pdf import PDFParser
from openviking.parse.parsers.text import TextParser
-from openviking.parse.parsers.code import CodeParser
if TYPE_CHECKING:
from openviking.parse.custom import CustomParserProtocol
@@ -53,6 +53,7 @@ class ParserRegistry:
if register_optional:
try:
from openviking.parse.parsers.media import ImageParser
+
self.register("image", ImageParser())
logger.info("Registered ImageParser for image formats")
except ImportError as e:
diff --git a/openviking/parse/resource_detector/__init__.py b/openviking/parse/resource_detector/__init__.py
index 48e37587e..2a6416107 100644
--- a/openviking/parse/resource_detector/__init__.py
+++ b/openviking/parse/resource_detector/__init__.py
@@ -1,4 +1,3 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
"""Detect resource type and understand the content before process."""
-
diff --git a/openviking/parse/resource_detector/detect_info.py b/openviking/parse/resource_detector/detect_info.py
index 2c301163c..30135cd93 100644
--- a/openviking/parse/resource_detector/detect_info.py
+++ b/openviking/parse/resource_detector/detect_info.py
@@ -7,6 +7,10 @@ We need to know:
"""
+from dataclasses import dataclass
+from enum import Enum
+
+
class VisitType(Enum):
# Direct content that can be used immediately, e.g., strings containing conversation content, JSON content, etc.
DIRECT_CONTENT = "DIRECT_CONTENT"
@@ -17,6 +21,7 @@ class VisitType(Enum):
# Pre-processed context pack conforming to OpenViking's structure, typically with .ovpack extension
READY_CONTEXT_PACK = "READY_CONTEXT_PACK"
+
class SizeType(Enum):
# Content can be processed directly in memory, e.g., small text segments
IN_MEM = "IN_MEM"
@@ -25,6 +30,7 @@ class SizeType(Enum):
# Content too large to process, e.g., exceeds X GB, may cause system crash or performance issues
TOO_LARGE_TO_PROCESS = "TOO_LARGE_TO_PROCESS"
+
class RecursiveType(Enum):
# Single file, no recursive processing required
SINGLE = "SINGLE"
@@ -33,6 +39,7 @@ class RecursiveType(Enum):
# Files that need to be expanded for recursive processing, e.g., compressed files, READY_CONTEXT_PACK, etc.
EXPAND_TO_RECURSIVE = "EXPAND_TO_RECURSIVE"
+
@dataclass
class DetectInfo:
visit_type: VisitType
diff --git a/openviking/parse/tree_builder.py b/openviking/parse/tree_builder.py
index 7d0960328..7bb6d4f9e 100644
--- a/openviking/parse/tree_builder.py
+++ b/openviking/parse/tree_builder.py
@@ -21,12 +21,11 @@ IMPORTANT (v5.0 Architecture):
"""
import logging
-from typing import TYPE_CHECKING, List, Optional, Tuple
-from openviking.utils.config import get_openviking_config
-from openviking.utils.uri import VikingURI
+from typing import TYPE_CHECKING, Optional
+
from openviking.core.building_tree import BuildingTree
-from openviking.prompts import render_prompt
from openviking.storage.viking_fs import get_viking_fs
+from openviking.utils.uri import VikingURI
if TYPE_CHECKING:
pass
@@ -206,9 +205,7 @@ class TreeBuilder:
if "exist" not in str(e).lower():
logger.debug(f"Parent dir {parent_uri} may already exist: {e}")
- async def _enqueue_semantic_generation(
- self, uri: str, context_type: str
- ) -> None:
+ async def _enqueue_semantic_generation(self, uri: str, context_type: str) -> None:
"""
Enqueue a directory for semantic generation.
@@ -216,16 +213,12 @@ class TreeBuilder:
uri: Directory URI to enqueue
context_type: resource/memory/skill
"""
- from openviking.storage.queuefs import get_queue_manager, SemanticMsg
+ from openviking.storage.queuefs import SemanticMsg, get_queue_manager
- viking_fs = get_viking_fs()
queue_manager = get_queue_manager()
# Get semantic queue
- semantic_queue = queue_manager.get_queue(
- queue_manager.SEMANTIC,
- allow_create=True
- )
+ semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)
# Sort by depth (descending) for bottom-up processing
msg = SemanticMsg(
@@ -251,8 +244,8 @@ class TreeBuilder:
if isinstance(result, str):
return result
elif isinstance(result, bytes):
- return result.decode('utf-8')
- elif hasattr(result, 'to_dict') and not isinstance(result, list):
+ return result.decode("utf-8")
+ elif hasattr(result, "to_dict") and not isinstance(result, list):
# Handle FindResult by converting to dict (skip lists)
return str(result.to_dict())
elif isinstance(result, list):
@@ -260,5 +253,3 @@ class TreeBuilder:
return json.dumps(result)
else:
return str(result)
-
-
diff --git a/openviking/parse/vlm.py b/openviking/parse/vlm.py
index 9eade9899..c410bea89 100644
--- a/openviking/parse/vlm.py
+++ b/openviking/parse/vlm.py
@@ -4,9 +4,10 @@
from dataclasses import dataclass
from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
-from openviking.utils.extractor import ImageInfo, TableInfo
+from typing import Any, Dict, List, Union
+
from openviking.prompts import render_prompt
+from openviking.utils.extractor import ImageInfo, TableInfo
from openviking.utils.llm import parse_json_from_response
from openviking.utils.logger import get_logger
@@ -52,6 +53,7 @@ class VLMProcessor:
def _get_vlm(self):
"""Get VLM singleton."""
from openviking.utils.config import get_openviking_config
+
return get_openviking_config().vlm
async def understand_image(
@@ -125,7 +127,6 @@ class VLMProcessor:
# Add sample data
if len(raw_data) > 1:
- sample_rows = min(3, len(raw_data) - 1)
overview_parts.append(f"Contains {len(raw_data) - 1} data records.")
overview = " ".join(overview_parts)
@@ -247,7 +248,9 @@ class VLMProcessor:
async def process_batch(batch_start: int, batch: List) -> tuple:
async with semaphore:
- results = await self._batch_understand_pages_single_call(batch, instruction, batch_start)
+ results = await self._batch_understand_pages_single_call(
+ batch, instruction, batch_start
+ )
return batch_start, results
# Run all batches concurrently
@@ -282,7 +285,7 @@ class VLMProcessor:
data = parse_json_from_response(response)
if not data or "pages" not in data:
- raise ValueError(f"Invalid VLM response: missing 'pages' field")
+ raise ValueError("Invalid VLM response: missing 'pages' field")
results = []
for page_info in data["pages"]:
@@ -350,7 +353,7 @@ class VLMProcessor:
images_section = ""
if images:
images_section = "\n".join(
- f"Image {i+1}: Located on page {img.page + 1}"
+ f"Image {i + 1}: Located on page {img.page + 1}"
for i, img in enumerate(images[: self.max_images_per_call])
)
else:
@@ -360,7 +363,7 @@ class VLMProcessor:
tables_section = ""
if tables:
tables_section = "\n".join(
- f"Table {i+1}: Located on page {tbl.page + 1}"
+ f"Table {i + 1}: Located on page {tbl.page + 1}"
for i, tbl in enumerate(tables[: self.max_images_per_call])
)
else:
@@ -370,7 +373,7 @@ class VLMProcessor:
sections_list = ""
if sections:
sections_list = "\n".join(
- f"Section {i+1}: {sec.get('title', 'Untitled')}"
+ f"Section {i + 1}: {sec.get('title', 'Untitled')}"
for i, sec in enumerate(sections[: self.max_sections_per_call])
)
else:
@@ -382,7 +385,9 @@ class VLMProcessor:
"title": title or "Unknown document",
"instruction": instruction or "Understand document content",
"reason": reason or "User added",
- "content_preview": content_preview[:2000] if content_preview else "No content preview",
+ "content_preview": content_preview[:2000]
+ if content_preview
+ else "No content preview",
"image_count": len(images),
"images_section": images_section,
"table_count": len(tables),
@@ -442,7 +447,9 @@ class VLMProcessor:
if table.has_structured_data():
result = self._understand_table_from_data(table, instruction)
else:
- result = VLMResult("[Table]", "Cannot parse table (VLM not available)", "[Table content]")
+ result = VLMResult(
+ "[Table]", "Cannot parse table (VLM not available)", "[Table content]"
+ )
table_results.append(result)
# Simplified prompt for text-only analysis
@@ -456,7 +463,7 @@ Content preview:
{content_preview[:3000]}
Section list:
-{chr(10).join(f"- {s.get('title', 'Untitled')}" for s in sections[:self.max_sections_per_call])}
+{chr(10).join(f"- {s.get('title', 'Untitled')}" for s in sections[: self.max_sections_per_call])}
Please output in JSON format:
{{
@@ -525,7 +532,7 @@ Please output in JSON format:
# Parse image results
image_results = []
image_data = data.get("images", [])
- for i, img in enumerate(images):
+ for i, _ in enumerate(images):
if i < len(image_data):
img_info = image_data[i]
result = VLMResult(
@@ -540,7 +547,7 @@ Please output in JSON format:
# Parse table results
table_results = []
table_data = data.get("tables", [])
- for i, tbl in enumerate(tables):
+ for i, _ in enumerate(tables):
if i < len(table_data):
tbl_info = table_data[i]
result = VLMResult(
@@ -607,11 +614,11 @@ Please output in JSON format:
images_info = []
image_data_list = []
- for i, (img_data, context) in enumerate(batch):
+ for i, (_img_data, context) in enumerate(batch):
images_info.append(
f"Image {i + 1}: Surrounding text: {context[:100] if context else 'No context'}"
)
- image_data_list.append(img_data)
+ image_data_list.append(_img_data)
prompt = render_prompt(
"vision.batch_filtering",
@@ -631,7 +638,7 @@ Please output in JSON format:
data = parse_json_from_response(response)
if data and "results" in data:
batch_results = []
- for i, (img_data, context) in enumerate(batch):
+ for i, (_img_data, _context) in enumerate(batch):
if i < len(data["results"]):
result = data["results"][i]
batch_results.append(
@@ -656,7 +663,10 @@ Please output in JSON format:
logger.error(f"Error filtering image batch: {e}")
# On error, keep all images by default
- return [{"is_meaningful": True, "reason": "Filtering failed", "image_type": "Unknown"} for _ in batch]
+ return [
+ {"is_meaningful": True, "reason": "Filtering failed", "image_type": "Unknown"}
+ for _ in batch
+ ]
async def _filter_single_image(
self,
diff --git a/openviking/retrieve/hierarchical_retriever.py b/openviking/retrieve/hierarchical_retriever.py
index 4d76e214a..6881e5691 100644
--- a/openviking/retrieve/hierarchical_retriever.py
+++ b/openviking/retrieve/hierarchical_retriever.py
@@ -8,40 +8,37 @@ and rerank-based relevance scoring.
"""
import heapq
-import re
-from typing import Any, Callable, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple
-from openviking.utils.config import RerankConfig
+from openviking.models.embedder.base import EmbedResult
from openviking.retrieve.types import (
ContextType,
MatchedContext,
QueryResult,
RelatedContext,
- ScoreDistribution,
- ThinkingTrace,
- TraceEventType,
TypedQuery,
)
from openviking.storage import VikingDBInterface
from openviking.storage.viking_fs import get_viking_fs
-from openviking.models.embedder.base import EmbedResult
+from openviking.utils.config import RerankConfig
from openviking.utils.logger import get_logger
-from openviking.utils.rerank import RerankClient
logger = get_logger(__name__)
+
class RetrieverMode(str):
THINKING = "thinking"
QUICK = "quick"
+
class HierarchicalRetriever:
"""Hierarchical retriever with dense and sparse vector support."""
- MAX_CONVERGENCE_ROUNDS = 3 # Stop after multiple rounds with unchanged topk
- MAX_RELATIONS = 5 # Maximum relations per resource
- SCORE_PROPAGATION_ALPHA = 0.5 # Score propagation coefficient
- DIRECTORY_DOMINANCE_RATIO = 1.2 # Directory score must exceed max child score
- GLOBAL_SEARCH_TOPK = 3 # Global retrieval count
+ MAX_CONVERGENCE_ROUNDS = 3 # Stop after multiple rounds with unchanged topk
+ MAX_RELATIONS = 5 # Maximum relations per resource
+ SCORE_PROPAGATION_ALPHA = 0.5 # Score propagation coefficient
+ DIRECTORY_DOMINANCE_RATIO = 1.2 # Directory score must exceed max child score
+ GLOBAL_SEARCH_TOPK = 3 # Global retrieval count
def __init__(
self,
@@ -67,10 +64,14 @@ class HierarchicalRetriever:
if rerank_config and rerank_config.is_available():
# TODO: Support later - initialize RerankClient here
self._rerank_client = None
- logger.info(f"[HierarchicalRetriever] Rerank config available, threshold={self.threshold}")
+ logger.info(
+ f"[HierarchicalRetriever] Rerank config available, threshold={self.threshold}"
+ )
else:
self._rerank_client = None
- logger.info(f"[HierarchicalRetriever] Rerank not configured, using vector search only with threshold={self.threshold}")
+ logger.info(
+ f"[HierarchicalRetriever] Rerank not configured, using vector search only with threshold={self.threshold}"
+ )
async def retrieve(
self,
@@ -105,10 +106,7 @@ class HierarchicalRetriever:
if metadata_filter:
filters_to_merge.append(metadata_filter)
- final_metadata_filter = {
- "op": "and",
- "conds": filters_to_merge
- }
+ final_metadata_filter = {"op": "and", "conds": filters_to_merge}
# Step 1: Determine starting directories based on context_type
root_uris = self._get_root_uris_for_type(query.context_type)
@@ -163,13 +161,16 @@ class HierarchicalRetriever:
return []
sparse_query_vector = result.sparse_vector or {}
- global_filter = {"op": "and", "conds": [filter, {"op": "must", "field": "is_leaf", "conds": [False]}]}
+ global_filter = {
+ "op": "and",
+ "conds": [filter, {"op": "must", "field": "is_leaf", "conds": [False]}],
+ }
results = await self.storage.search(
collection=collection,
query_vector=query_vector,
sparse_query_vector=sparse_query_vector,
filter=global_filter,
- limit=limit
+ limit=limit,
)
return results
@@ -192,7 +193,7 @@ class HierarchicalRetriever:
if self._rerank_client and mode == RetrieverMode.THINKING:
for r in global_results:
# todo: multi-modal
- doc = r['abstract']
+ doc = r["abstract"]
docs.append(doc)
rerank_scores = self._rerank_client.rerank_batch(query, docs)
for i, r in enumerate(global_results):
@@ -244,10 +245,7 @@ class HierarchicalRetriever:
"""Merge filter conditions."""
if not extra_filter:
return base_filter
- return {
- "op": "and",
- "conds": [base_filter, extra_filter]
- }
+ return {"op": "and", "conds": [base_filter, extra_filter]}
# Generate query vectors
query_vector = None
@@ -288,7 +286,9 @@ class HierarchicalRetriever:
collection=collection,
query_vector=query_vector,
sparse_query_vector=sparse_query_vector, # Pass sparse vector
- filter=merge_filter({'op': 'must', "field": "parent_uri", "conds":[current_uri]}, metadata_filter),
+ filter=merge_filter(
+ {"op": "must", "field": "parent_uri", "conds": [current_uri]}, metadata_filter
+ ),
limit=pre_filter_limit,
)
@@ -300,7 +300,7 @@ class HierarchicalRetriever:
documents = []
for r in results:
# todo: multi-modal
- doc = r['abstract']
+ doc = r["abstract"]
documents.append(doc)
rerank_scores = self._rerank_client.rerank_batch(query, documents)
@@ -311,23 +311,29 @@ class HierarchicalRetriever:
for r, score in zip(results, query_scores):
uri = r.get("uri", "")
- final_score = alpha * score + (1 - alpha) * current_score if current_score else score
+ final_score = (
+ alpha * score + (1 - alpha) * current_score if current_score else score
+ )
if passes_threshold(final_score) and uri not in visited:
r["_final_score"] = final_score
collected.append(r)
visited.add(uri)
- logger.info(f"[RecursiveSearch] Added URI: {uri} to candidates with score: {final_score}")
+ logger.info(
+ f"[RecursiveSearch] Added URI: {uri} to candidates with score: {final_score}"
+ )
if not r.get("is_leaf"):
continue
heapq.heappush(dir_queue, (-final_score, uri))
else:
- logger.info(f"[RecursiveSearch] URI {uri} score {final_score} did not pass threshold {effective_threshold}")
+ logger.info(
+ f"[RecursiveSearch] URI {uri} score {final_score} did not pass threshold {effective_threshold}"
+ )
# Convergence check
- current_topk = sorted(
- collected, key=lambda x: x.get("_final_score", 0), reverse=True
- )[:limit]
+ current_topk = sorted(collected, key=lambda x: x.get("_final_score", 0), reverse=True)[
+ :limit
+ ]
current_topk_uris = {c.get("uri", "") for c in current_topk}
if current_topk_uris == prev_topk_uris and len(current_topk_uris) >= limit:
diff --git a/openviking/retrieve/intent_analyzer.py b/openviking/retrieve/intent_analyzer.py
index 1a6cb95cb..0d5b4725f 100644
--- a/openviking/retrieve/intent_analyzer.py
+++ b/openviking/retrieve/intent_analyzer.py
@@ -6,12 +6,12 @@ Intent analyzer for OpenViking retrieval.
Analyzes session context to generate query plans.
"""
-from typing import Any, Dict, List, Optional
+from typing import List, Optional
-from openviking.utils.config import get_openviking_config
from openviking.message import Message
from openviking.prompts import render_prompt
from openviking.retrieve.types import ContextType, QueryPlan, TypedQuery
+from openviking.utils.config import get_openviking_config
from openviking.utils.llm import parse_json_from_response
from openviking.utils.logger import get_logger
@@ -59,9 +59,7 @@ class IntentAnalyzer:
)
# Call LLM
- response = await get_openviking_config().vlm.get_completion_async(
- prompt
- )
+ response = await get_openviking_config().vlm.get_completion_async(prompt)
# Parse result
parsed = parse_json_from_response(response)
@@ -88,7 +86,7 @@ class IntentAnalyzer:
# Log analysis result
for i, q in enumerate(queries):
logger.info(
- f' [{i+1}] type={q.context_type.value}, priority={q.priority}, query="{q.query}"'
+ f' [{i + 1}] type={q.context_type.value}, priority={q.priority}, query="{q.query}"'
)
logger.debug(f"[IntentAnalyzer] Reasoning: {parsed.get('reasoning', '')[:200]}...")
@@ -111,11 +109,9 @@ class IntentAnalyzer:
summary = compression_summary if compression_summary else "None"
# Format recent messages
- recent = messages[-self.max_recent_messages:] if messages else []
+ recent = messages[-self.max_recent_messages :] if messages else []
recent_messages = (
- "\n".join(f"[{m.role}]: {m.content}" for m in recent if m.content)
- if recent
- else "None"
+ "\n".join(f"[{m.role}]: {m.content}" for m in recent if m.content) if recent else "None"
)
# Current message
diff --git a/openviking/retrieve/types.py b/openviking/retrieve/types.py
index a38eef660..04a50e4b7 100644
--- a/openviking/retrieve/types.py
+++ b/openviking/retrieve/types.py
@@ -272,6 +272,7 @@ class QueryPlan:
@dataclass
class RelatedContext:
"""Related context with summary."""
+
uri: str
abstract: str
diff --git a/openviking/session/compressor.py b/openviking/session/compressor.py
index e36574a3a..8c4981dc8 100644
--- a/openviking/session/compressor.py
+++ b/openviking/session/compressor.py
@@ -8,11 +8,10 @@ Uses MemoryExtractor for 6-category extraction and MemoryDeduplicator for LLM-ba
"""
from dataclasses import dataclass
-from typing import Any, Dict, List, Optional
+from typing import Dict, List
from openviking.core.context import Context
from openviking.message import Message
-from openviking.utils.config import get_openviking_config
from openviking.storage import VikingDBManager
from openviking.storage.viking_fs import get_viking_fs
from openviking.utils import get_logger
@@ -48,6 +47,7 @@ class SessionCompressor:
async def _index_memory(self, memory: Context) -> bool:
"""Add memory to vectorization queue."""
from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter
+
embedding_msg = EmbeddingMsgConverter.from_context(memory)
await self.vikingdb.enqueue_embedding_msg(embedding_msg)
logger.info(f"Enqueued memory for vectorization: {memory.uri}")
@@ -149,16 +149,22 @@ class SessionCompressor:
for memory_uri in memory_uris:
if resource_uris:
await viking_fs.link(
- memory_uri, resource_uris, reason="Memory extracted from session using these resources"
+ memory_uri,
+ resource_uris,
+ reason="Memory extracted from session using these resources",
)
if skill_uris:
await viking_fs.link(
- memory_uri, skill_uris, reason="Memory extracted from session calling these skills"
+ memory_uri,
+ skill_uris,
+ reason="Memory extracted from session calling these skills",
)
# Resources/skills -> memories (reverse)
for resource_uri in resource_uris:
- await viking_fs.link(resource_uri, memory_uris, reason="Referenced by these memories")
+ await viking_fs.link(
+ resource_uri, memory_uris, reason="Referenced by these memories"
+ )
for skill_uri in skill_uris:
await viking_fs.link(skill_uri, memory_uris, reason="Called by these memories")
diff --git a/openviking/session/memory_deduplicator.py b/openviking/session/memory_deduplicator.py
index 64dad6cf3..266efda68 100644
--- a/openviking/session/memory_deduplicator.py
+++ b/openviking/session/memory_deduplicator.py
@@ -8,16 +8,16 @@ LLM-assisted deduplication with CREATE/UPDATE/MERGE/SKIP decisions (Mem0 inspire
from dataclasses import dataclass
from enum import Enum
-from typing import Any, Dict, List, Optional
+from typing import List, Optional
-from openviking.utils.config import get_openviking_config
-from openviking.core.context import Context, ContextType
+from openviking.core.context import Context
+from openviking.models.embedder.base import EmbedResult
from openviking.prompts import render_prompt
from openviking.storage import VikingDBManager
from openviking.utils import get_logger
-from openviking.models.embedder.base import EmbedResult
+from openviking.utils.config import get_openviking_config
-from .memory_extractor import CandidateMemory, MemoryCategory
+from .memory_extractor import CandidateMemory
logger = get_logger(__name__)
diff --git a/openviking/session/memory_extractor.py b/openviking/session/memory_extractor.py
index bb23fe0c8..6c664af19 100644
--- a/openviking/session/memory_extractor.py
+++ b/openviking/session/memory_extractor.py
@@ -9,17 +9,15 @@ Extracts 6 categories of memories from session:
"""
from dataclasses import dataclass
-from datetime import datetime
from enum import Enum
-from typing import Any, List, Optional
+from typing import List, Optional
from uuid import uuid4
-from openviking.utils.config import get_openviking_config
from openviking.core.context import Context, ContextType, Vectorize
-from openviking.message import Message
from openviking.prompts import render_prompt
from openviking.storage.viking_fs import get_viking_fs
from openviking.utils import get_logger
+from openviking.utils.config import get_openviking_config
logger = get_logger(__name__)
@@ -82,11 +80,7 @@ class MemoryExtractor:
# Format all messages
messages = context["messages"]
- formatted_messages = "\n".join([
- f"[{m.role}]: {m.content}"
- for m in messages
- if m.content
- ])
+ formatted_messages = "\n".join([f"[{m.role}]: {m.content}" for m in messages if m.content])
if not formatted_messages:
return []
diff --git a/openviking/session/session.py b/openviking/session/session.py
index 46bd36a6f..764e40c42 100644
--- a/openviking/session/session.py
+++ b/openviking/session/session.py
@@ -13,9 +13,9 @@ from datetime import datetime
from typing import TYPE_CHECKING, Any, Dict, List, Optional
from uuid import uuid4
-from openviking.utils.config import get_openviking_config
from openviking.message import Message, Part
from openviking.utils import get_logger
+from openviking.utils.config import get_openviking_config
if TYPE_CHECKING:
from openviking.sync_client import OpenViking
@@ -29,6 +29,7 @@ def _run_async(coro):
loop = asyncio.get_running_loop()
# If already in event loop, use nest_asyncio or return directly
import nest_asyncio
+
nest_asyncio.apply()
return loop.run_until_complete(coro)
except RuntimeError:
@@ -102,9 +103,9 @@ class Session:
return
try:
- content = _run_async(self.client.viking_fs.read_file(
- f"{self._session_uri}/messages.jsonl"
- ))
+ content = _run_async(
+ self.client.viking_fs.read_file(f"{self._session_uri}/messages.jsonl")
+ )
self._messages = [
Message.from_dict(json.loads(line))
for line in content.strip().split("\n")
@@ -117,7 +118,9 @@ class Session:
# Restore compression_index (scan history directory)
try:
history_items = _run_async(self.client.viking_fs.ls(f"{self._session_uri}/history"))
- archives = [item["name"] for item in history_items if item["name"].startswith("archive_")]
+ archives = [
+ item["name"] for item in history_items if item["name"].startswith("archive_")
+ ]
if archives:
max_index = max(int(a.split("_")[1]) for a in archives)
self._compression.compression_index = max_index
@@ -245,11 +248,13 @@ class Session:
logger.info(
f"Starting memory extraction from {len(messages_to_archive)} archived messages"
)
- memories = _run_async(self.client._session_compressor.extract_long_term_memories(
- messages=messages_to_archive,
- user=self.user,
- session_id=self.session_id,
- ))
+ memories = _run_async(
+ self.client._session_compressor.extract_long_term_memories(
+ messages=messages_to_archive,
+ user=self.user,
+ session_id=self.session_id,
+ )
+ )
logger.info(f"Extracted {len(memories)} memories")
result["memories_extracted"] = len(memories)
self._stats.memories_extracted += len(memories)
@@ -287,11 +292,13 @@ class Session:
for usage in self._usage_records:
try:
- _run_async(storage.update(
- collection="context",
- filter={"uri": usage.uri},
- update={"$inc": {"active_count": 1}},
- ))
+ _run_async(
+ storage.update(
+ collection="context",
+ filter={"uri": usage.uri},
+ update={"$inc": {"active_count": 1}},
+ )
+ )
updated += 1
except Exception as e:
logger.debug(f"Could not update active_count for {usage.uri}: {e}")
@@ -321,9 +328,7 @@ class Session:
summaries = []
if self.compression.compression_index > 0:
try:
- history_items = _run_async(self.client.viking_fs.ls(
- f"{self._session_uri}/history"
- ))
+ history_items = _run_async(self.client.viking_fs.ls(f"{self._session_uri}/history"))
query_lower = query.lower()
# Collect all archives with relevance scores
@@ -409,10 +414,12 @@ class Session:
# Write messages.jsonl
lines = [m.to_jsonl() for m in messages]
- _run_async(viking_fs.write_file(
- uri=f"{archive_uri}/messages.jsonl",
- content="\n".join(lines) + "\n",
- ))
+ _run_async(
+ viking_fs.write_file(
+ uri=f"{archive_uri}/messages.jsonl",
+ content="\n".join(lines) + "\n",
+ )
+ )
_run_async(viking_fs.write_file(uri=f"{archive_uri}/.abstract.md", content=abstract))
_run_async(viking_fs.write_file(uri=f"{archive_uri}/.overview.md", content=overview))
@@ -433,29 +440,37 @@ class Session:
lines = [m.to_jsonl() for m in messages]
content = "\n".join(lines) + "\n" if lines else ""
- _run_async(viking_fs.write_file(
- uri=f"{self._session_uri}/messages.jsonl",
- content=content,
- ))
+ _run_async(
+ viking_fs.write_file(
+ uri=f"{self._session_uri}/messages.jsonl",
+ content=content,
+ )
+ )
# Update L0/L1
- _run_async(viking_fs.write_file(
- uri=f"{self._session_uri}/.abstract.md",
- content=abstract,
- ))
- _run_async(viking_fs.write_file(
- uri=f"{self._session_uri}/.overview.md",
- content=overview,
- ))
+ _run_async(
+ viking_fs.write_file(
+ uri=f"{self._session_uri}/.abstract.md",
+ content=abstract,
+ )
+ )
+ _run_async(
+ viking_fs.write_file(
+ uri=f"{self._session_uri}/.overview.md",
+ content=overview,
+ )
+ )
def _append_to_jsonl(self, msg: Message) -> None:
"""Append to messages.jsonl."""
if not self.client or not self.client.viking_fs:
return
- _run_async(self.client.viking_fs.append_file(
- f"{self._session_uri}/messages.jsonl",
- msg.to_jsonl() + "\n",
- ))
+ _run_async(
+ self.client.viking_fs.append_file(
+ f"{self._session_uri}/messages.jsonl",
+ msg.to_jsonl() + "\n",
+ )
+ )
def _update_message_in_jsonl(self) -> None:
"""Update message in messages.jsonl."""
@@ -464,10 +479,12 @@ class Session:
lines = [m.to_jsonl() for m in self._messages]
content = "\n".join(lines) + "\n"
- _run_async(self.client.viking_fs.write_file(
- f"{self._session_uri}/messages.jsonl",
- content,
- ))
+ _run_async(
+ self.client.viking_fs.write_file(
+ f"{self._session_uri}/messages.jsonl",
+ content,
+ )
+ )
def _save_tool_result(
self,
@@ -493,10 +510,12 @@ class Session:
"status": status,
"time": {"created": datetime.now().isoformat()},
}
- _run_async(self.client.viking_fs.write_file(
- f"{self._session_uri}/tools/{tool_id}/tool.json",
- json.dumps(tool_data, ensure_ascii=False),
- ))
+ _run_async(
+ self.client.viking_fs.write_file(
+ f"{self._session_uri}/tools/{tool_id}/tool.json",
+ json.dumps(tool_data, ensure_ascii=False),
+ )
+ )
def _generate_abstract(self) -> str:
"""Generate one-sentence summary for session."""
@@ -516,7 +535,9 @@ class Session:
f"- `messages.jsonl` - Current messages ({turn_count} turns)",
]
if self._compression.compression_index > 0:
- parts.append(f"- `history/` - Historical archives ({self._compression.compression_index} total)")
+ parts.append(
+ f"- `history/` - Historical archives ({self._compression.compression_index} total)"
+ )
parts.extend(
[
"",
diff --git a/openviking/storage/__init__.py b/openviking/storage/__init__.py
index b3cffd909..636207224 100644
--- a/openviking/storage/__init__.py
+++ b/openviking/storage/__init__.py
@@ -2,6 +2,10 @@
# SPDX-License-Identifier: Apache-2.0
"""Storage layer interfaces and implementations."""
+from openviking.storage.observers import BaseObserver, QueueObserver
+from openviking.storage.queuefs import QueueManager, get_queue_manager, init_queue_manager
+from openviking.storage.viking_fs import VikingFS, get_viking_fs, init_viking_fs
+from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend
from openviking.storage.vikingdb_interface import (
CollectionNotFoundError,
ConnectionError,
@@ -11,16 +15,7 @@ from openviking.storage.vikingdb_interface import (
StorageException,
VikingDBInterface,
)
-
-from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend
from openviking.storage.vikingdb_manager import VikingDBManager
-from openviking.storage.queuefs import QueueManager, get_queue_manager, init_queue_manager
-from openviking.storage.viking_fs import (
- VikingFS,
- get_viking_fs,
- init_viking_fs,
-)
-from openviking.storage.observers import BaseObserver, QueueObserver
__all__ = [
# Interface
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.py
index 7e1ab2905..6b51fb929 100644
--- a/openviking/storage/collection_schemas.py
+++ b/openviking/storage/collection_schemas.py
@@ -7,20 +7,21 @@ Provides centralized schema definitions and factory functions for creating colle
similar to how init_viking_fs encapsulates VikingFS initialization.
"""
-import abc
import json
-from typing import Dict, Any, Optional
-from openviking.storage.queuefs.named_queue import DequeueHandlerBase
+from typing import Any, Dict, Optional
+
+from openviking.models.embedder.base import EmbedResult
from openviking.storage.queuefs.embedding_msg import EmbeddingMsg
+from openviking.storage.queuefs.named_queue import DequeueHandlerBase
from openviking.storage.vikingdb_interface import VikingDBInterface
from openviking.utils import get_logger
from openviking.utils.config.open_viking_config import OpenVikingConfig
-from openviking.models.embedder.base import EmbedResult
logger = get_logger(__name__)
COLLECTION_NAME = "context"
+
class CollectionSchemas:
"""
Centralized collection schema definitions.
@@ -72,12 +73,14 @@ async def init_context_collection(storage) -> bool:
True if collection was created, False if already exists
"""
from openviking.utils.config import get_openviking_config
+
config = get_openviking_config()
name = config.storage.vectordb.name
vector_dim = config.embedding.dimension
schema = CollectionSchemas.context_collection(name, vector_dim)
return await storage.create_collection(name, schema)
+
class TextEmbeddingHandler(DequeueHandlerBase):
"""
Text embedding handler that converts text messages to embedding vectors
@@ -97,6 +100,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):
vikingdb: VikingDBInterface instance for writing to vector database
"""
from openviking.utils.config import get_openviking_config
+
self._vikingdb = vikingdb
self._embedder = None
config = get_openviking_config()
@@ -128,6 +132,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):
# Initialize embedder if not already initialized
if not self._embedder:
from openviking.utils.config import get_openviking_config
+
config = get_openviking_config()
self._initialize_embedder(config)
@@ -159,10 +164,13 @@ class TextEmbeddingHandler(DequeueHandlerBase):
try:
record_id = await self._vikingdb.insert(self._collection_name, inserted_data)
if record_id:
- logger.debug(f"Successfully wrote embedding to database: {record_id} abstract {inserted_data['abstract']} vector {inserted_data['vector'][:5]}")
+ logger.debug(
+ f"Successfully wrote embedding to database: {record_id} abstract {inserted_data['abstract']} vector {inserted_data['vector'][:5]}"
+ )
except Exception as db_err:
logger.error(f"Failed to write to vector database: {db_err}")
import traceback
+
traceback.print_exc()
self.report_error(str(db_err), data)
return None
@@ -173,6 +181,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):
except Exception as e:
logger.error(f"Error processing embedding message: {e}")
import traceback
+
traceback.print_exc()
self.report_error(str(e), data)
return None
diff --git a/openviking/storage/observers/vlm_observer.py b/openviking/storage/observers/vlm_observer.py
index 28dc709e5..561c98823 100644
--- a/openviking/storage/observers/vlm_observer.py
+++ b/openviking/storage/observers/vlm_observer.py
@@ -6,8 +6,8 @@ VLMObserver: VLM system observability tool.
Provides methods to observe and report token usage across VLM models and backends.
"""
-from openviking.storage.observers.base_observer import BaseObserver
from openviking.models.vlm.base import VLMBase
+from openviking.storage.observers.base_observer import BaseObserver
from openviking.utils.logger import get_logger
logger = get_logger(__name__)
diff --git a/openviking/storage/queuefs/__init__.py b/openviking/storage/queuefs/__init__.py
index f9e19339e..7016da0ec 100644
--- a/openviking/storage/queuefs/__init__.py
+++ b/openviking/storage/queuefs/__init__.py
@@ -2,11 +2,11 @@
# SPDX-License-Identifier: Apache-2.0
from .embedding_msg import EmbeddingMsg
from .embedding_queue import EmbeddingQueue
+from .named_queue import NamedQueue, QueueError, QueueStatus
+from .queue_manager import QueueManager, get_queue_manager, init_queue_manager
from .semantic_msg import SemanticMsg
-from .semantic_queue import SemanticQueue
from .semantic_processor import SemanticProcessor
-from .named_queue import NamedQueue, QueueStatus, QueueError
-from .queue_manager import QueueManager, get_queue_manager, init_queue_manager
+from .semantic_queue import SemanticQueue
__all__ = [
"QueueManager",
diff --git a/openviking/storage/queuefs/embedding_msg.py b/openviking/storage/queuefs/embedding_msg.py
index e599a8445..19b8381ea 100644
--- a/openviking/storage/queuefs/embedding_msg.py
+++ b/openviking/storage/queuefs/embedding_msg.py
@@ -1,9 +1,9 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
import json
+from dataclasses import asdict, dataclass
+from typing import Any, Dict, List, Union
from uuid import uuid4
-from dataclasses import dataclass, asdict
-from typing import List, Dict, Any, Optional, Union
@dataclass
@@ -11,9 +11,7 @@ class EmbeddingMsg:
message: Union[str, List[Dict[str, Any]]]
context_data: Dict[str, Any]
- def __init__(
- self, message: Union[str, List[Dict[str, Any]]], context_data: Dict[str, Any]
- ):
+ def __init__(self, message: Union[str, List[Dict[str, Any]]], context_data: Dict[str, Any]):
self.id = str(uuid4())
self.message = message
self.context_data = context_data
@@ -21,11 +19,11 @@ class EmbeddingMsg:
def to_dict(self) -> Dict[str, Any]:
"""Convert embedding message to dictionary format."""
return asdict(self)
-
+
def to_json(self) -> str:
"""Convert embedding message to JSON string."""
return json.dumps(self.to_dict())
-
+
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "EmbeddingMsg":
"""Create an embedding message object from dictionary."""
@@ -43,4 +41,4 @@ class EmbeddingMsg:
data = json.loads(json_str)
return cls.from_dict(data)
except json.JSONDecodeError as e:
- raise ValueError(f"Invalid JSON string: {e}")
\ No newline at end of file
+ raise ValueError(f"Invalid JSON string: {e}")
diff --git a/openviking/storage/queuefs/embedding_msg_converter.py b/openviking/storage/queuefs/embedding_msg_converter.py
index 577257fe3..3c70ec780 100644
--- a/openviking/storage/queuefs/embedding_msg_converter.py
+++ b/openviking/storage/queuefs/embedding_msg_converter.py
@@ -7,8 +7,6 @@ This module provides a unified interface for converting Context objects
to EmbeddingMsg objects for asynchronous vector processing.
"""
-import json
-
from openviking.core.context import Context
from openviking.storage.queuefs.embedding_msg import EmbeddingMsg
from openviking.utils import get_logger
@@ -36,5 +34,3 @@ class EmbeddingMsgConverter:
if hasattr(embedding_msg.context_data, key) and value is not None:
setattr(embedding_msg.context_data, key, value)
return embedding_msg
-
-
diff --git a/openviking/storage/queuefs/embedding_queue.py b/openviking/storage/queuefs/embedding_queue.py
index 06f23e5a8..57ad0e892 100644
--- a/openviking/storage/queuefs/embedding_queue.py
+++ b/openviking/storage/queuefs/embedding_queue.py
@@ -1,15 +1,15 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
-import json
-from dataclasses import asdict
-from typing import Any, Dict, Optional, Union
+from typing import Optional
-from .named_queue import NamedQueue
-from .embedding_msg import EmbeddingMsg
from openviking.utils.logger import get_logger
+from .embedding_msg import EmbeddingMsg
+from .named_queue import NamedQueue
+
logger = get_logger(__name__)
+
class EmbeddingQueue(NamedQueue):
"""EmbeddingQueue: Named queue specifically for processing EmbeddingMsg.
@@ -43,13 +43,13 @@ class EmbeddingQueue(NamedQueue):
data_dict = await super().peek()
if not data_dict:
return None
-
+
if "data" in data_dict and isinstance(data_dict["data"], str):
try:
return EmbeddingMsg.from_json(data_dict["data"])
except Exception:
return None
-
+
try:
return EmbeddingMsg.from_dict(data_dict)
except Exception:
diff --git a/openviking/storage/queuefs/named_queue.py b/openviking/storage/queuefs/named_queue.py
index ee0b7eca4..046e9c76b 100644
--- a/openviking/storage/queuefs/named_queue.py
+++ b/openviking/storage/queuefs/named_queue.py
@@ -6,7 +6,7 @@ import json
import threading
from dataclasses import dataclass, field
from datetime import datetime
-from typing import Any, Dict, List, Optional, Union, Callable, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
from openviking.utils.logger import get_logger
@@ -19,6 +19,7 @@ logger = get_logger(__name__)
@dataclass
class QueueError:
"""Error record."""
+
timestamp: datetime
message: str
data: Optional[Dict[str, Any]] = None
@@ -27,6 +28,7 @@ class QueueError:
@dataclass
class QueueStatus:
"""Queue status."""
+
pending: int = 0
in_progress: int = 0
processed: int = 0
@@ -138,13 +140,15 @@ class NamedQueue:
with self._lock:
self._in_progress -= 1
self._error_count += 1
- self._errors.append(QueueError(
- timestamp=datetime.now(),
- message=error_msg,
- data=data,
- ))
+ self._errors.append(
+ QueueError(
+ timestamp=datetime.now(),
+ message=error_msg,
+ data=data,
+ )
+ )
if len(self._errors) > self.MAX_ERRORS:
- self._errors = self._errors[-self.MAX_ERRORS:]
+ self._errors = self._errors[-self.MAX_ERRORS :]
async def get_status(self) -> QueueStatus:
"""Get queue status."""
@@ -194,7 +198,9 @@ class NamedQueue:
data = json.dumps(data)
loop = asyncio.get_event_loop()
- msg_id = await loop.run_in_executor(None, self._agfs.write, enqueue_file, data.encode("utf-8"))
+ msg_id = await loop.run_in_executor(
+ None, self._agfs.write, enqueue_file, data.encode("utf-8")
+ )
return msg_id if isinstance(msg_id, str) else str(msg_id)
async def dequeue(self) -> Optional[Dict[str, Any]]:
@@ -223,7 +229,7 @@ class NamedQueue:
"""Peek at head message without removing."""
await self._ensure_initialized()
peek_file = f"{self.path}/peek"
-
+
loop = asyncio.get_event_loop()
try:
content = await loop.run_in_executor(None, self._agfs.read, peek_file)
@@ -238,7 +244,7 @@ class NamedQueue:
"""Get queue size."""
await self._ensure_initialized()
size_file = f"{self.path}/size"
-
+
loop = asyncio.get_event_loop()
try:
content = await loop.run_in_executor(None, self._agfs.read, size_file)
@@ -253,7 +259,7 @@ class NamedQueue:
"""Clear queue."""
await self._ensure_initialized()
clear_file = f"{self.path}/clear"
-
+
loop = asyncio.get_event_loop()
try:
await loop.run_in_executor(None, self._agfs.write, clear_file, b"")
diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.py
index 6f40df455..7d3a76ff8 100644
--- a/openviking/storage/queuefs/queue_manager.py
+++ b/openviking/storage/queuefs/queue_manager.py
@@ -5,15 +5,16 @@ QueueManager: Encapsulates AGFS QueueFS plugin operations.
All queues are managed through NamedQueue.
"""
-import time
import asyncio
import atexit
import threading
-from typing import Any, Dict, List, Optional, Union
+import time
+from typing import Any, Dict, Optional, Union
from openviking.utils.logger import get_logger
-from .named_queue import NamedQueue, EnqueueHookBase, DequeueHandlerBase, QueueStatus
+
from .embedding_queue import EmbeddingQueue
+from .named_queue import DequeueHandlerBase, EnqueueHookBase, NamedQueue, QueueStatus
from .semantic_queue import SemanticQueue
logger = get_logger(__name__)
diff --git a/openviking/storage/queuefs/semantic_msg.py b/openviking/storage/queuefs/semantic_msg.py
index 899e215f1..e4d30be82 100644
--- a/openviking/storage/queuefs/semantic_msg.py
+++ b/openviking/storage/queuefs/semantic_msg.py
@@ -3,15 +3,16 @@
"""SemanticMsg: Semantic extraction queue message dataclass."""
import json
-from uuid import uuid4
-from dataclasses import dataclass, asdict, field
-from typing import List, Dict, Any, Optional
+from dataclasses import asdict, dataclass
from datetime import datetime
+from typing import Any, Dict
+from uuid import uuid4
@dataclass
class SemanticMsg:
"""Semantic extraction queue message."""
+
id: str # UUID
uri: str # Directory URI
context_type: str # resource, memory, skill
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index dcaabfc67..7e300666a 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -3,16 +3,16 @@
"""SemanticProcessor: Processes messages from SemanticQueue, generates .abstract.md and .overview.md."""
import asyncio
-from typing import List, Dict, Any, Tuple
+from typing import Any, Dict, List, Tuple
-from openviking.utils import VikingURI
-from openviking.utils.logger import get_logger
-from openviking.utils.config import get_openviking_config
+from openviking.core.context import Context, ResourceContentType, Vectorize
from openviking.prompts import render_prompt
-from openviking.storage.viking_fs import get_viking_fs
from openviking.storage.queuefs.named_queue import DequeueHandlerBase
from openviking.storage.queuefs.semantic_msg import SemanticMsg
-from openviking.core.context import Context, Vectorize, ResourceContentType
+from openviking.storage.viking_fs import get_viking_fs
+from openviking.utils import VikingURI
+from openviking.utils.config import get_openviking_config
+from openviking.utils.logger import get_logger
logger = get_logger(__name__)
@@ -77,6 +77,7 @@ class SemanticProcessor(DequeueHandlerBase):
"""Process dequeued SemanticMsg, recursively process all subdirectories."""
try:
import json
+
if "data" in data and isinstance(data["data"], str):
data = json.loads(data["data"])
@@ -146,9 +147,7 @@ class SemanticProcessor(DequeueHandlerBase):
except Exception as e:
logger.error(f"Failed to vectorize files in {uri}: {e}", exc_info=True)
- async def _collect_children_abstracts(
- self, children_uris: List[str]
- ) -> List[Dict[str, str]]:
+ async def _collect_children_abstracts(self, children_uris: List[str]) -> List[Dict[str, str]]:
"""Collect .abstract.md from subdirectories."""
viking_fs = get_viking_fs()
results = []
@@ -159,9 +158,7 @@ class SemanticProcessor(DequeueHandlerBase):
results.append({"name": dir_name, "abstract": abstract})
return results
- async def _generate_file_summaries(
- self, file_paths: List[str]
- ) -> List[Dict[str, str]]:
+ async def _generate_file_summaries(self, file_paths: List[str]) -> List[Dict[str, str]]:
"""Concurrently generate file summaries."""
if not file_paths:
return []
@@ -218,26 +215,26 @@ class SemanticProcessor(DequeueHandlerBase):
def _extract_abstract_from_overview(self, overview_content: str) -> str:
"""Extract abstract from overview.md."""
- lines = overview_content.split('\n')
+ lines = overview_content.split("\n")
# Skip header lines (starting with #)
content_lines = []
in_header = True
for line in lines:
- if in_header and line.startswith('#'):
+ if in_header and line.startswith("#"):
continue
elif in_header and line.strip():
in_header = False
if not in_header:
# Stop at first ##
- if line.startswith('##'):
+ if line.startswith("##"):
break
if line.strip():
content_lines.append(line.strip())
- return '\n'.join(content_lines).strip()
+ return "\n".join(content_lines).strip()
async def _generate_overview(
self,
@@ -256,6 +253,7 @@ class SemanticProcessor(DequeueHandlerBase):
Overview content
"""
import re
+
vlm = get_openviking_config().vlm
if not vlm.is_available():
@@ -266,14 +264,16 @@ class SemanticProcessor(DequeueHandlerBase):
file_index_map = {}
file_summaries_lines = []
for idx, item in enumerate(file_summaries, 1):
- file_index_map[idx] = item['name']
+ file_index_map[idx] = item["name"]
file_summaries_lines.append(f"[{idx}] {item['name']}: {item['summary']}")
file_summaries_str = "\n".join(file_summaries_lines) if file_summaries_lines else "None"
# Build subdirectory summary string
- children_abstracts_str = "\n".join(
- f"- {item['name']}/: {item['abstract']}" for item in children_abstracts
- ) if children_abstracts else "None"
+ children_abstracts_str = (
+ "\n".join(f"- {item['name']}/: {item['abstract']}" for item in children_abstracts)
+ if children_abstracts
+ else "None"
+ )
# Generate overview
try:
@@ -293,7 +293,7 @@ class SemanticProcessor(DequeueHandlerBase):
idx = int(match.group(1))
return file_index_map.get(idx, match.group(0))
- overview = re.sub(r'\[(\d+)\]', replace_index, overview)
+ overview = re.sub(r"\[(\d+)\]", replace_index, overview)
return overview.strip()
@@ -305,9 +305,9 @@ class SemanticProcessor(DequeueHandlerBase):
self, uri: str, context_type: str, abstract: str, overview: str
) -> None:
"""Create directory Context and enqueue to EmbeddingQueue."""
+
from openviking.storage.queuefs import get_queue_manager
from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter
- from datetime import datetime
parent_uri = VikingURI(uri).parent.uri
context = Context(
@@ -315,7 +315,7 @@ class SemanticProcessor(DequeueHandlerBase):
parent_uri=parent_uri,
is_leaf=False,
abstract=abstract,
- context_type=context_type
+ context_type=context_type,
)
context.set_vectorize(Vectorize(text=overview))
@@ -333,10 +333,11 @@ class SemanticProcessor(DequeueHandlerBase):
file_summaries: List[Dict[str, str]],
) -> None:
"""Vectorize files in directory."""
+ from datetime import datetime
+
from openviking.core.context import Context
from openviking.storage.queuefs import get_queue_manager
from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter
- from datetime import datetime
queue_manager = get_queue_manager()
embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)
@@ -361,7 +362,7 @@ class SemanticProcessor(DequeueHandlerBase):
embedding_msg = EmbeddingMsgConverter.from_context(context)
await embedding_queue.enqueue(embedding_msg)
logger.debug(f"Enqueued file for vectorization: {file_path}")
-
+
def get_resource_content_type(self, file_name: str) -> ResourceContentType:
def _is_image_file(file_name: str) -> bool:
image_extensions = {".png", ".jpg", ".jpeg", ".gif", ".bmp", ".svg", ".webp"}
@@ -370,11 +371,11 @@ class SemanticProcessor(DequeueHandlerBase):
def _is_video_file(file_name: str) -> bool:
video_extensions = {".mp4", ".avi", ".mov", ".wmv", ".flv"}
return any(file_name.endswith(ext) for ext in video_extensions)
-
+
def _is_text_file(file_name: str) -> bool:
text_extensions = {".txt", ".md", ".csv", ".json", ".xml"}
return any(file_name.endswith(ext) for ext in text_extensions)
-
+
def _is_audio_file(file_name: str) -> bool:
audio_extensions = {".mp3", ".wav", ".aac", ".flac"}
return any(file_name.endswith(ext) for ext in audio_extensions)
@@ -387,5 +388,5 @@ class SemanticProcessor(DequeueHandlerBase):
return ResourceContentType.VIDEO
elif _is_audio_file(file_name):
return ResourceContentType.AUDIO
-
+
return ResourceContentType.TEXT
diff --git a/openviking/storage/queuefs/semantic_queue.py b/openviking/storage/queuefs/semantic_queue.py
index feb1edbbc..d758f3bee 100644
--- a/openviking/storage/queuefs/semantic_queue.py
+++ b/openviking/storage/queuefs/semantic_queue.py
@@ -5,6 +5,7 @@
from typing import Optional
from openviking.utils.logger import get_logger
+
from .named_queue import NamedQueue
from .semantic_msg import SemanticMsg
diff --git a/openviking/storage/vectordb/__init__.py b/openviking/storage/vectordb/__init__.py
index 8ec4a0931..34a1a0f9f 100644
--- a/openviking/storage/vectordb/__init__.py
+++ b/openviking/storage/vectordb/__init__.py
@@ -1,4 +1,3 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
"""VectorDB storage backend."""
-
diff --git a/openviking/storage/vectordb/collection/collection.py b/openviking/storage/vectordb/collection/collection.py
index a15804450..45954e8cb 100644
--- a/openviking/storage/vectordb/collection/collection.py
+++ b/openviking/storage/vectordb/collection/collection.py
@@ -193,9 +193,9 @@ class Collection:
Raises:
AssertionError: If the provided `collection` is not an instance of ICollection.
"""
- assert isinstance(
- collection, ICollection
- ), "collection must be an instance of CollectionInterface"
+ assert isinstance(collection, ICollection), (
+ "collection must be an instance of CollectionInterface"
+ )
self.__collection: Optional[ICollection] = collection
def __del__(self):
@@ -392,9 +392,7 @@ class Collection:
"""
if self.__collection is None:
raise RuntimeError("Collection is closed")
- return self.__collection.search_by_id(
- index_name, id, limit, offset, filters, output_fields
- )
+ return self.__collection.search_by_id(index_name, id, limit, offset, filters, output_fields)
def search_by_multimodal(
self,
@@ -459,9 +457,7 @@ class Collection:
"""
if self.__collection is None:
raise RuntimeError("Collection is closed")
- return self.__collection.search_by_random(
- index_name, limit, offset, filters, output_fields
- )
+ return self.__collection.search_by_random(index_name, limit, offset, filters, output_fields)
def search_by_scalar(
self,
diff --git a/openviking/storage/vectordb/collection/local_collection.py b/openviking/storage/vectordb/collection/local_collection.py
index 99cb7cfdc..43604f838 100644
--- a/openviking/storage/vectordb/collection/local_collection.py
+++ b/openviking/storage/vectordb/collection/local_collection.py
@@ -26,24 +26,21 @@ from openviking.storage.vectordb.meta.collection_meta import CollectionMeta, cre
from openviking.storage.vectordb.meta.index_meta import create_index_meta
from openviking.storage.vectordb.store.data import CandidateData, DeltaRecord
from openviking.storage.vectordb.store.store import OpType
-from openviking.storage.vectordb.store.store_manager import (
- StoreManager,
- create_store_manager,
-)
+from openviking.storage.vectordb.store.store_manager import StoreManager, create_store_manager
from openviking.storage.vectordb.utils import validation
+from openviking.storage.vectordb.utils.config_utils import get_config_value
from openviking.storage.vectordb.utils.constants import (
DEFAULT_INDEX_MAINTENANCE_SECONDS,
DEFAULT_TTL_CLEANUP_SECONDS,
ENV_INDEX_MAINTENANCE_SECONDS,
ENV_TTL_CLEANUP_SECONDS,
+ STORAGE_DIR_NAME,
AggregateKeys,
SpecialFields,
- STORAGE_DIR_NAME,
)
from openviking.storage.vectordb.utils.dict_utils import ThreadSafeDictManager
from openviking.storage.vectordb.utils.id_generator import generate_auto_id
from openviking.storage.vectordb.utils.str_to_uint64 import str_to_uint64
-from openviking.storage.vectordb.utils.config_utils import get_config_value
from openviking.storage.vectordb.vectorize.base import BaseVectorizer
from openviking.storage.vectordb.vectorize.vectorizer import VectorizerAdapter
from openviking.storage.vectordb.vectorize.vectorizer_factory import VectorizerFactory
@@ -902,7 +899,9 @@ class PersistCollection(LocalCollection):
for index_name in index_names:
meta_path = os.path.join(self.index_dir, index_name, "index_meta.json")
if not os.path.exists(meta_path):
- logger.warning(f"Index metadata file not found at {meta_path}, skipping recovery for index {index_name}")
+ logger.warning(
+ f"Index metadata file not found at {meta_path}, skipping recovery for index {index_name}"
+ )
continue
meta = create_index_meta(self.meta, meta_path)
# When recovering an existing index, pass initial_timestamp=0.
diff --git a/openviking/storage/vectordb/collection/volcengine_collection.py b/openviking/storage/vectordb/collection/volcengine_collection.py
index 7dac57dd5..7ddfa309e 100644
--- a/openviking/storage/vectordb/collection/volcengine_collection.py
+++ b/openviking/storage/vectordb/collection/volcengine_collection.py
@@ -20,9 +20,7 @@ from openviking.storage.vectordb.collection.volcengine_clients import (
from openviking.utils.logger import default_logger as logger
-def get_or_create_volcengine_collection(
- config: Dict[str, Any], meta_data: Dict[str, Any]
-):
+def get_or_create_volcengine_collection(config: Dict[str, Any], meta_data: Dict[str, Any]):
"""
Get or create a Volcengine Collection.
diff --git a/openviking/storage/vectordb/index/local_index.py b/openviking/storage/vectordb/index/local_index.py
index 0c642ff6f..4132b19ec 100644
--- a/openviking/storage/vectordb/index/local_index.py
+++ b/openviking/storage/vectordb/index/local_index.py
@@ -11,7 +11,6 @@ from typing import Any, Dict, List, Optional, Tuple, Union
import openviking.storage.vectordb.engine as engine
from openviking.storage.vectordb.index.index import IIndex
from openviking.storage.vectordb.store.data import CandidateData, DeltaRecord
-from openviking.storage.vectordb.utils import file_utils
from openviking.storage.vectordb.utils.constants import IndexFileMarkers
from openviking.utils.logger import default_logger as logger
@@ -85,15 +84,15 @@ class IndexEngineProxy:
query_vector = normalize_vector(query_vector)
req.query = query_vector
req.topk = limit
-
+
if filters is None:
filters = {}
req.dsl = json.dumps(filters)
-
+
if sparse_raw_terms and sparse_values:
req.sparse_raw_terms = sparse_raw_terms
req.sparse_values = sparse_values
-
+
search_result = self.index_engine.search(req)
labels = search_result.labels
scores = search_result.scores
@@ -102,7 +101,7 @@ class IndexEngineProxy:
def add_data(self, cands_list: List[CandidateData]):
if not self.index_engine:
raise RuntimeError("Index engine not initialized")
-
+
add_req_list = [engine.AddDataRequest() for _ in range(len(cands_list))]
for i, data in enumerate(cands_list):
add_req_list[i].label = data.label
@@ -255,7 +254,7 @@ class LocalIndex(IIndex):
sparse_raw_terms = []
if sparse_values is None:
sparse_values = []
-
+
return self.engine_proxy.search(
query_vector, limit, filters, sparse_raw_terms, sparse_values
)
@@ -267,7 +266,7 @@ class LocalIndex(IIndex):
) -> Dict[str, Any]:
if not self.engine_proxy or not self.engine_proxy.index_engine:
return {}
-
+
extra_json = ""
try:
req = engine.SearchRequest()
@@ -366,7 +365,7 @@ class VolatileIndex(LocalIndex):
"""
if cands_list is None:
cands_list = []
-
+
index_config_dict = meta.get_build_index_dict()
version_int = int(time.time_ns())
index_config_dict["VectorIndex"]["ElementCount"] = len(cands_list)
@@ -438,13 +437,13 @@ class PersistentIndex(LocalIndex):
"""
def __init__(
- self,
- name: str,
- meta: Any,
- path: str,
- cands_list: Optional[List[CandidateData]] = None,
+ self,
+ name: str,
+ meta: Any,
+ path: str,
+ cands_list: Optional[List[CandidateData]] = None,
force_rebuild: bool = False,
- initial_timestamp: Optional[int] = None
+ initial_timestamp: Optional[int] = None,
):
"""Initialize a persistent index with versioning support.
@@ -473,12 +472,12 @@ class PersistentIndex(LocalIndex):
"""
if cands_list is None:
cands_list = []
-
+
self.index_dir = os.path.join(path, name)
os.makedirs(self.index_dir, exist_ok=True)
self.version_dir = os.path.join(self.index_dir, "versions")
os.makedirs(self.version_dir, exist_ok=True)
-
+
newest_version = self.get_newest_version()
# At this point, there is no index, need to create a new one
@@ -491,11 +490,17 @@ class PersistentIndex(LocalIndex):
super().__init__(index_path, meta)
# Remove scheduling logic, unified scheduling by collection layer
- def _create_new_index(self, name: str, meta: Any, cands_list: List[CandidateData], initial_timestamp: Optional[int] = None):
+ def _create_new_index(
+ self,
+ name: str,
+ meta: Any,
+ cands_list: List[CandidateData],
+ initial_timestamp: Optional[int] = None,
+ ):
"""Create a new index from scratch."""
# Get the vector normalization flag from meta
normalize_vector_flag = meta.inner_meta.get("VectorIndex", {}).get("NormalizeVector", False)
-
+
version_int = initial_timestamp if initial_timestamp is not None else int(time.time_ns())
version_str = str(version_int)
index_config_dict = meta.get_build_index_dict()
@@ -503,11 +508,11 @@ class PersistentIndex(LocalIndex):
index_config_dict["VectorIndex"]["MaxElementCount"] = len(cands_list)
index_config_dict["UpdateTimeStamp"] = version_int
index_config_json = json.dumps(index_config_dict)
-
+
builder = IndexEngineProxy(index_config_json, normalize_vector_flag)
build_index_path = os.path.join(self.version_dir, version_str)
builder.add_data(cands_list)
-
+
dump_version_int = builder.dump(build_index_path)
if dump_version_int > 0:
dump_version_str = str(dump_version_int)
@@ -645,21 +650,21 @@ class PersistentIndex(LocalIndex):
# Must be a directory
if not os.path.isdir(version_path):
continue
-
+
# Must be an integer (timestamp)
if not name.isdigit():
continue
-
+
# Must have corresponding .write_done file
marker_path = version_path + IndexFileMarkers.WRITE_DONE.value
if not os.path.exists(marker_path):
continue
-
+
valid_versions.append(int(name))
-
+
if not valid_versions:
return 0
-
+
return max(valid_versions)
def drop(self):
diff --git a/openviking/storage/vectordb/meta/collection_meta.py b/openviking/storage/vectordb/meta/collection_meta.py
index 0c217c759..59a5ceb51 100644
--- a/openviking/storage/vectordb/meta/collection_meta.py
+++ b/openviking/storage/vectordb/meta/collection_meta.py
@@ -1,7 +1,7 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
import copy
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Optional
from openviking.storage.vectordb.meta.dict import IDict
from openviking.storage.vectordb.meta.local_dict import PersistentDict, VolatileDict
@@ -102,11 +102,7 @@ class CollectionMeta:
"",
)
inner_meta["SparseVectorKey"] = next(
- (
- item["FieldName"]
- for item in fields
- if item.get("FieldType") == "sparse_vector"
- ),
+ (item["FieldName"] for item in fields if item.get("FieldType") == "sparse_vector"),
"",
)
return inner_meta
@@ -124,9 +120,7 @@ class CollectionMeta:
user_meta = copy.deepcopy(inner_meta)
if not user_meta.get("HasPrimaryKey", False):
fields = user_meta.get("Fields", [])
- new_list = [
- item for item in fields if "AUTO_ID" != item.get("FieldName", "")
- ]
+ new_list = [item for item in fields if "AUTO_ID" != item.get("FieldName", "")]
user_meta["Fields"] = new_list
user_meta.pop("HasPrimaryKey", None)
user_meta.pop("FieldsCount", None)
diff --git a/openviking/storage/vectordb/meta/index_meta.py b/openviking/storage/vectordb/meta/index_meta.py
index 1704c2cc2..3c2253e72 100644
--- a/openviking/storage/vectordb/meta/index_meta.py
+++ b/openviking/storage/vectordb/meta/index_meta.py
@@ -29,9 +29,7 @@ def create_index_meta(
"""
inner_meta = {}
if user_meta:
- if not validation.is_valid_index_meta_data(
- user_meta, collection_meta.fields_dict
- ):
+ if not validation.is_valid_index_meta_data(user_meta, collection_meta.fields_dict):
raise ValueError(
"invalid index_meta {} fields_dict {}".format(
user_meta, collection_meta.fields_dict
@@ -120,9 +118,7 @@ class IndexMeta:
if "flat" in inner_meta["VectorIndex"]["IndexType"]:
vector_index["IndexType"] = "flat"
if "EnableSparse" in inner_meta["VectorIndex"]:
- vector_index["EnableSparse"] = inner_meta[
- "VectorIndex"
- ]["EnableSparse"]
+ vector_index["EnableSparse"] = inner_meta["VectorIndex"]["EnableSparse"]
if "SearchWithSparseLogitAlpha" in inner_meta["VectorIndex"]:
vector_index["SearchWithSparseLogitAlpha"] = inner_meta["VectorIndex"][
"SearchWithSparseLogitAlpha"
diff --git a/openviking/storage/vectordb/project/project.py b/openviking/storage/vectordb/project/project.py
index e06949822..1ed5d13c0 100644
--- a/openviking/storage/vectordb/project/project.py
+++ b/openviking/storage/vectordb/project/project.py
@@ -1,165 +1,166 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
-class Project:
- """Wrapper class for managing project operations.
+from abc import ABC, abstractmethod
+from typing import Any, Dict
- A Project serves as a container for managing multiple collections. It provides
- a unified interface for creating, accessing, and managing collections within
- a project namespace.
+
+class IProject(ABC):
+ """Interface defining the contract for project implementations.
+
+ All project implementations must inherit from this interface and implement
+ all abstract methods for managing collections.
"""
- def __init__(self, project):
- """Initialize the Project wrapper.
+ def __init__(self, project_name: str = "default"):
+ """Initialize the project interface.
Args:
- project (IProject): An instance implementing the IProject interface.
-
- Raises:
- AssertionError: If project is not an instance of IProject.
+ project_name (str): Name of the project. Defaults to 'default'.
"""
- assert isinstance(project, IProject), "project must be IProject"
- self.__project = project
+ self.project_name = project_name
+ @abstractmethod
def close(self):
- """Close the project and release all associated resources."""
- self.__project.close()
+ """Close the project and release resources.
- def has_collection(self, collection_name):
- """Check if a collection exists in the project.
+ Must be implemented by subclasses to properly clean up resources.
+ """
+ pass
+
+ @abstractmethod
+ def has_collection(self, collection_name: str) -> bool:
+ """Check if a collection exists.
Args:
collection_name (str): Name of the collection to check.
Returns:
- bool: True if the collection exists, False otherwise.
+ bool: True if collection exists, False otherwise.
"""
- return self.__project.has_collection(collection_name)
+ pass
- def get_collection(self, collection_name):
+ @abstractmethod
+ def get_collection(self, collection_name: str) -> Any:
"""Retrieve a collection by name.
Args:
collection_name (str): Name of the collection to retrieve.
Returns:
- Collection: The requested collection instance, or None if not found.
+ Collection: The collection instance, or None if not found.
"""
- return self.__project.get_collection(collection_name)
+ pass
- def get_collections(self):
+ @abstractmethod
+ def get_collections(self) -> Dict[str, Any]:
"""Get all collections in the project.
Returns:
- Dict[str, Collection]: Dictionary mapping collection names to Collection instances.
+ Dict[str, Collection]: Mapping of collection names to Collection instances.
"""
- return self.__project.get_collections()
+ pass
- def create_collection(self, collection_name, collection_meta):
- """Create a new collection in the project.
+ @abstractmethod
+ def create_collection(self, collection_name: str, collection_meta: Dict[str, Any]) -> Any:
+ """Create a new collection.
Args:
- collection_name (str): Name for the new collection. Must be unique within the project.
- collection_meta (Dict[str, Any]): Metadata configuration for the collection, including
- fields definition, primary key, vector configuration, etc.
+ collection_name (str): Unique name for the collection.
+ collection_meta (Dict[str, Any]): Collection metadata and configuration.
Returns:
Collection: The newly created collection instance.
-
- Raises:
- ValueError: If a collection with the same name already exists or if the metadata is invalid.
"""
- return self.__project.create_collection(collection_name, collection_meta)
+ pass
- def drop_collection(self, collection_name):
- """Delete a collection from the project.
+ @abstractmethod
+ def drop_collection(self, collection_name: str):
+ """Delete a collection.
Args:
collection_name (str): Name of the collection to delete.
-
- Note:
- This operation is irreversible and will permanently delete all data and indexes
- associated with the collection.
"""
- return self.__project.drop_collection(collection_name)
-
+ pass
-from abc import ABC, abstractmethod
-from typing import Any, Dict, Optional
-class IProject(ABC):
- """Interface defining the contract for project implementations.
+class Project:
+ """Wrapper class for managing project operations.
- All project implementations must inherit from this interface and implement
- all abstract methods for managing collections.
+ A Project serves as a container for managing multiple collections. It provides
+ a unified interface for creating, accessing, and managing collections within
+ a project namespace.
"""
- def __init__(self, project_name: str = "default"):
- """Initialize the project interface.
+ def __init__(self, project):
+ """Initialize the Project wrapper.
Args:
- project_name (str): Name of the project. Defaults to 'default'.
+ project (IProject): An instance implementing the IProject interface.
+
+ Raises:
+ AssertionError: If project is not an instance of IProject.
"""
- self.project_name = project_name
+ assert isinstance(project, IProject), "project must be IProject"
+ self.__project = project
- @abstractmethod
def close(self):
- """Close the project and release resources.
-
- Must be implemented by subclasses to properly clean up resources.
- """
- pass
+ """Close the project and release all associated resources."""
+ self.__project.close()
- @abstractmethod
- def has_collection(self, collection_name: str) -> bool:
- """Check if a collection exists.
+ def has_collection(self, collection_name):
+ """Check if a collection exists in the project.
Args:
collection_name (str): Name of the collection to check.
Returns:
- bool: True if collection exists, False otherwise.
+ bool: True if the collection exists, False otherwise.
"""
- pass
+ return self.__project.has_collection(collection_name)
- @abstractmethod
- def get_collection(self, collection_name: str) -> Any:
+ def get_collection(self, collection_name):
"""Retrieve a collection by name.
Args:
collection_name (str): Name of the collection to retrieve.
Returns:
- Collection: The collection instance, or None if not found.
+ Collection: The requested collection instance, or None if not found.
"""
- pass
+ return self.__project.get_collection(collection_name)
- @abstractmethod
- def get_collections(self) -> Dict[str, Any]:
+ def get_collections(self):
"""Get all collections in the project.
Returns:
- Dict[str, Collection]: Mapping of collection names to Collection instances.
+ Dict[str, Collection]: Dictionary mapping collection names to Collection instances.
"""
- pass
+ return self.__project.get_collections()
- @abstractmethod
- def create_collection(self, collection_name: str, collection_meta: Dict[str, Any]) -> Any:
- """Create a new collection.
+ def create_collection(self, collection_name, collection_meta):
+ """Create a new collection in the project.
Args:
- collection_name (str): Unique name for the collection.
- collection_meta (Dict[str, Any]): Collection metadata and configuration.
+ collection_name (str): Name for the new collection. Must be unique within the project.
+ collection_meta (Dict[str, Any]): Metadata configuration for the collection, including
+ fields definition, primary key, vector configuration, etc.
Returns:
Collection: The newly created collection instance.
+
+ Raises:
+ ValueError: If a collection with the same name already exists or if the metadata is invalid.
"""
- pass
+ return self.__project.create_collection(collection_name, collection_meta)
- @abstractmethod
- def drop_collection(self, collection_name: str):
- """Delete a collection.
+ def drop_collection(self, collection_name):
+ """Delete a collection from the project.
Args:
collection_name (str): Name of the collection to delete.
+
+ Note:
+ This operation is irreversible and will permanently delete all data and indexes
+ associated with the collection.
"""
- pass
+ return self.__project.drop_collection(collection_name)
diff --git a/openviking/storage/vectordb/service/api_fastapi.py b/openviking/storage/vectordb/service/api_fastapi.py
index aae163c5e..52230b93e 100644
--- a/openviking/storage/vectordb/service/api_fastapi.py
+++ b/openviking/storage/vectordb/service/api_fastapi.py
@@ -5,8 +5,7 @@ import time
from dataclasses import asdict
from typing import Any, Optional
-from fastapi import APIRouter, Query, Request, Depends, HTTPException
-from fastapi.responses import JSONResponse
+from fastapi import APIRouter, Depends, Query, Request
from openviking.storage.vectordb.project.project_group import get_or_create_project_group
from openviking.storage.vectordb.service.app_models import (
@@ -28,7 +27,6 @@ from openviking.storage.vectordb.service.app_models import (
)
from openviking.storage.vectordb.service.code import ErrorCode
from openviking.storage.vectordb.utils import data_utils
-from openviking.storage.vectordb.utils.api_utils import form_error, form_res
from openviking.utils.logger import default_logger as logger
@@ -85,34 +83,38 @@ search_router = APIRouter(prefix="/api/vikingdb/data/search", tags=["Search"])
# ==================== Dependencies ====================
+
def get_project(project_name: str = "default"):
"""Get project instance"""
return project_group.get_or_create_project(project_name)
+
def get_collection_or_raise(collection_name: str, project_name: str = "default"):
"""Get collection instance or raise exception if not found"""
if not collection_name:
raise VikingDBException(ErrorCode.INVALID_PARAM, "collection name is empty")
-
+
project = project_group.get_or_create_project(project_name)
collection = project.get_collection(collection_name)
if not collection:
raise VikingDBException(ErrorCode.COLLECTION_NOT_EXIST, "collection not exist")
return collection
+
# Dependency for GET requests using Query parameters
def get_collection_dependency(
CollectionName: str = Query(..., description="Collection name"),
- ProjectName: Optional[str] = Query("default", description="Project name")
+ ProjectName: Optional[str] = Query("default", description="Project name"),
):
if not CollectionName:
raise VikingDBException(ErrorCode.INVALID_PARAM, "collection name is empty")
return get_collection_or_raise(CollectionName, ProjectName)
+
# Dependency for snake_case query params
def get_collection_dependency_snake(
collection_name: str = Query(..., description="Collection name"),
- project: Optional[str] = Query("default", description="Project name")
+ project: Optional[str] = Query("default", description="Project name"),
):
if not collection_name:
raise VikingDBException(ErrorCode.INVALID_PARAM, "collection name is empty")
@@ -164,8 +166,12 @@ async def update_collection(request: CollectionUpdateRequest, req: Request):
"""Update an existing collection"""
try:
if not request.CollectionName:
- return error_response("CollectionName is empty", ErrorCode.INVALID_PARAM.value, request=req)
- collection = get_collection_or_raise(request.CollectionName, request.ProjectName or "default")
+ return error_response(
+ "CollectionName is empty", ErrorCode.INVALID_PARAM.value, request=req
+ )
+ collection = get_collection_or_raise(
+ request.CollectionName, request.ProjectName or "default"
+ )
description = request.Description
fields = data_utils.convert_dict(request.Fields)
collection.update(fields, description)
@@ -175,10 +181,7 @@ async def update_collection(request: CollectionUpdateRequest, req: Request):
@collection_router.get("/GetVikingdbCollection", response_model=ApiResponse)
-async def get_collection_info(
- req: Request,
- collection: Any = Depends(get_collection_dependency)
-):
+async def get_collection_info(req: Request, collection: Any = Depends(get_collection_dependency)):
"""Get collection information"""
meta_data = collection.get_meta_data()
return success_response("collection info", meta_data, request=req)
@@ -229,7 +232,7 @@ async def upsert_data(request: DataUpsertRequest, req: Request):
"""Upsert data to collection"""
try:
collection = get_collection_or_raise(request.collection_name, request.project or "default")
-
+
ttl = request.ttl or 0
data_list = data_utils.convert_dict(request.fields)
@@ -247,7 +250,7 @@ async def upsert_data(request: DataUpsertRequest, req: Request):
async def fetch_data(
req: Request,
ids: str = Query(..., description="Primary key list"),
- collection: Any = Depends(get_collection_dependency_snake)
+ collection: Any = Depends(get_collection_dependency_snake),
):
"""Fetch data from collection"""
primary_keys = data_utils.convert_dict(ids)
@@ -260,7 +263,7 @@ async def delete_data(request: DataDeleteRequest, req: Request):
"""Delete data from collection"""
try:
collection = get_collection_or_raise(request.collection_name, request.project or "default")
-
+
primary_keys = data_utils.convert_dict(request.ids) if request.ids else []
del_all = request.del_all or False
@@ -281,8 +284,10 @@ async def delete_data(request: DataDeleteRequest, req: Request):
async def create_index(request: IndexCreateRequest, req: Request):
"""Create an index"""
try:
- collection = get_collection_or_raise(request.CollectionName, request.ProjectName or "default")
-
+ collection = get_collection_or_raise(
+ request.CollectionName, request.ProjectName or "default"
+ )
+
index_name = request.IndexName
if not index_name:
return error_response("index name is empty", ErrorCode.INVALID_PARAM.value, request=req)
@@ -312,8 +317,10 @@ async def create_index(request: IndexCreateRequest, req: Request):
async def update_index(request: IndexUpdateRequest, req: Request):
"""Update an index"""
try:
- collection = get_collection_or_raise(request.CollectionName, request.ProjectName or "default")
-
+ collection = get_collection_or_raise(
+ request.CollectionName, request.ProjectName or "default"
+ )
+
index_name = request.IndexName
if not index_name:
return error_response("index name is empty", ErrorCode.INVALID_PARAM.value, request=req)
@@ -331,7 +338,7 @@ async def update_index(request: IndexUpdateRequest, req: Request):
async def get_index_info(
req: Request,
IndexName: str = Query(..., description="Index name"),
- collection: Any = Depends(get_collection_dependency)
+ collection: Any = Depends(get_collection_dependency),
):
"""Get index information"""
if not IndexName:
@@ -342,10 +349,7 @@ async def get_index_info(
@index_router.get("/ListVikingdbIndex", response_model=ApiResponse)
-async def list_indexes(
- req: Request,
- collection: Any = Depends(get_collection_dependency)
-):
+async def list_indexes(req: Request, collection: Any = Depends(get_collection_dependency)):
"""List all indexes"""
data = collection.list_indexes()
return success_response("list indexes success", data, request=req)
@@ -355,8 +359,10 @@ async def list_indexes(
async def drop_index(request: IndexDropRequest, req: Request):
"""Delete an index"""
try:
- collection = get_collection_or_raise(request.CollectionName, request.ProjectName or "default")
-
+ collection = get_collection_or_raise(
+ request.CollectionName, request.ProjectName or "default"
+ )
+
index_name = request.IndexName
if not index_name:
return error_response("index name is empty", ErrorCode.INVALID_PARAM.value, request=req)
@@ -375,7 +381,7 @@ async def search_by_vector(request: SearchByVectorRequest, req: Request):
"""Search by vector"""
try:
collection = get_collection_or_raise(request.collection_name, request.project or "default")
-
+
index_name = request.index_name
if not index_name:
return error_response("index name is empty", ErrorCode.INVALID_PARAM.value, request=req)
@@ -392,7 +398,9 @@ async def search_by_vector(request: SearchByVectorRequest, req: Request):
offset = request.offset or 0
if offset < 0:
return error_response(
- "offset must be greater than or equal to 0", ErrorCode.INVALID_PARAM.value, request=req
+ "offset must be greater than or equal to 0",
+ ErrorCode.INVALID_PARAM.value,
+ request=req,
)
result = collection.search_by_vector(
@@ -414,7 +422,7 @@ async def search_by_id(request: SearchByIdRequest, req: Request):
"""Search by ID"""
try:
collection = get_collection_or_raise(request.collection_name, request.project or "default")
-
+
index_name = request.index_name
if not index_name:
return error_response("index name is empty", ErrorCode.INVALID_PARAM.value, request=req)
@@ -433,7 +441,9 @@ async def search_by_id(request: SearchByIdRequest, req: Request):
offset = request.offset or 0
if offset < 0:
return error_response(
- "offset must be greater than or equal to 0", ErrorCode.INVALID_PARAM.value, request=req
+ "offset must be greater than or equal to 0",
+ ErrorCode.INVALID_PARAM.value,
+ request=req,
)
result = collection.search_by_id(
@@ -454,7 +464,7 @@ async def search_by_multimodal(request: SearchByMultiModalRequest, req: Request)
"""Search by multimodal"""
try:
collection = get_collection_or_raise(request.collection_name, request.project or "default")
-
+
index_name = request.index_name
if not index_name:
return error_response("index name is empty", ErrorCode.INVALID_PARAM.value, request=req)
@@ -480,7 +490,9 @@ async def search_by_multimodal(request: SearchByMultiModalRequest, req: Request)
offset = request.offset or 0
if offset < 0:
return error_response(
- "offset must be greater than or equal to 0", ErrorCode.INVALID_PARAM.value, request=req
+ "offset must be greater than or equal to 0",
+ ErrorCode.INVALID_PARAM.value,
+ request=req,
)
try:
@@ -507,7 +519,7 @@ async def search_by_scalar(request: SearchByScalarRequest, req: Request):
"""Search by scalar field"""
try:
collection = get_collection_or_raise(request.collection_name, request.project or "default")
-
+
index_name = request.index_name
if not index_name:
return error_response("index name is empty", ErrorCode.INVALID_PARAM.value, request=req)
@@ -527,7 +539,9 @@ async def search_by_scalar(request: SearchByScalarRequest, req: Request):
offset = request.offset or 0
if offset < 0:
return error_response(
- "offset must be greater than or equal to 0", ErrorCode.INVALID_PARAM.value, request=req
+ "offset must be greater than or equal to 0",
+ ErrorCode.INVALID_PARAM.value,
+ request=req,
)
result = collection.search_by_scalar(
@@ -549,7 +563,7 @@ async def search_by_random(request: SearchByRandomRequest, req: Request):
"""Search by random"""
try:
collection = get_collection_or_raise(request.collection_name, request.project or "default")
-
+
index_name = request.index_name
if not index_name:
return error_response("index name is empty", ErrorCode.INVALID_PARAM.value, request=req)
@@ -564,7 +578,9 @@ async def search_by_random(request: SearchByRandomRequest, req: Request):
offset = request.offset or 0
if offset < 0:
return error_response(
- "offset must be greater than or equal to 0", ErrorCode.INVALID_PARAM.value, request=req
+ "offset must be greater than or equal to 0",
+ ErrorCode.INVALID_PARAM.value,
+ request=req,
)
result = collection.search_by_random(
@@ -584,7 +600,7 @@ async def search_by_keywords(request: SearchByKeywordsRequest, req: Request):
"""Search by keywords"""
try:
collection = get_collection_or_raise(request.collection_name, request.project or "default")
-
+
index_name = request.index_name
if not index_name:
return error_response("index name is empty", ErrorCode.INVALID_PARAM.value, request=req)
@@ -609,7 +625,9 @@ async def search_by_keywords(request: SearchByKeywordsRequest, req: Request):
offset = request.offset or 0
if offset < 0:
return error_response(
- "offset must be greater than or equal to 0", ErrorCode.INVALID_PARAM.value, request=req
+ "offset must be greater than or equal to 0",
+ ErrorCode.INVALID_PARAM.value,
+ request=req,
)
try:
diff --git a/openviking/storage/vectordb/service/server_fastapi.py b/openviking/storage/vectordb/service/server_fastapi.py
index fa010537d..d63914974 100644
--- a/openviking/storage/vectordb/service/server_fastapi.py
+++ b/openviking/storage/vectordb/service/server_fastapi.py
@@ -45,8 +45,7 @@ app = FastAPI(
@app.exception_handler(VikingDBException)
async def vikingdb_exception_handler(request: Request, exc: VikingDBException):
return JSONResponse(
- status_code=200,
- content=error_response(exc.message, exc.code.value, request=request)
+ status_code=200, content=error_response(exc.message, exc.code.value, request=request)
)
diff --git a/openviking/storage/vectordb/store/bytes_row.py b/openviking/storage/vectordb/store/bytes_row.py
index 043a638a3..61eb178df 100644
--- a/openviking/storage/vectordb/store/bytes_row.py
+++ b/openviking/storage/vectordb/store/bytes_row.py
@@ -211,7 +211,7 @@ class BytesRow:
field_meta = self.schema.get_field_meta(field_name)
if field_meta.id >= serialized_data[0]:
return field_meta.default_value
-
+
# Use '<' for little-endian in all unpack operations
if field_meta.data_type == FieldType.int64:
return struct.unpack_from("<q", serialized_data, field_meta.offset)[0]
diff --git a/openviking/storage/vectordb/store/local_store.py b/openviking/storage/vectordb/store/local_store.py
index 106a0dc8c..b1ab54cb5 100644
--- a/openviking/storage/vectordb/store/local_store.py
+++ b/openviking/storage/vectordb/store/local_store.py
@@ -1,13 +1,13 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
-import time
-from typing import List, Tuple, Union, Any
+from typing import List, Tuple, Union
import openviking.storage.vectordb.engine as engine
from openviking.storage.vectordb.store.store import BatchOp, IMutiTableStore, Op, OpType
# Constant for the maximum Unicode character, used for range queries to cover all possible keys
-MAX_UNICODE_CHAR = "\U0010FFFF"
+MAX_UNICODE_CHAR = "\U0010ffff"
+
def create_store_engine_proxy(path: str = "") -> "StoreEngineProxy":
"""Create a storage engine proxy.
@@ -175,9 +175,7 @@ class StoreEngineProxy(IMutiTableStore):
# batch_op.op_type can be a list or a single value
if isinstance(batch_op.op_type, list):
op_type = (
- batch_op.op_type[i]
- if i < len(batch_op.op_type)
- else batch_op.op_type[0]
+ batch_op.op_type[i] if i < len(batch_op.op_type) else batch_op.op_type[0]
)
else:
op_type = batch_op.op_type
@@ -189,8 +187,6 @@ class StoreEngineProxy(IMutiTableStore):
engine_op.key = batch_op.table + key
# Safety check for data_list
- engine_op.value = (
- batch_op.data_list[i] if i < len(batch_op.data_list) else ""
- )
+ engine_op.value = batch_op.data_list[i] if i < len(batch_op.data_list) else ""
engine_op_list.append(engine_op)
self.storage_engine.exec_op(engine_op_list)
diff --git a/openviking/storage/vectordb/store/serializable.py b/openviking/storage/vectordb/store/serializable.py
index 185464c20..f2939a63d 100644
--- a/openviking/storage/vectordb/store/serializable.py
+++ b/openviking/storage/vectordb/store/serializable.py
@@ -3,9 +3,11 @@
"""
Elegant serialization decorator: automatically generate schema and serialization methods from type annotations
"""
-from dataclasses import fields, is_dataclass, asdict
-from typing import Any, get_origin, get_args
-from openviking.storage.vectordb.store.bytes_row import Schema, BytesRow, FieldType
+
+from dataclasses import asdict, fields, is_dataclass
+from typing import Any, get_args, get_origin
+
+from openviking.storage.vectordb.store.bytes_row import BytesRow, FieldType, Schema
def _python_type_to_field_type(py_type: Any, field_name: str) -> FieldType:
diff --git a/openviking/storage/vectordb/store/store_manager.py b/openviking/storage/vectordb/store/store_manager.py
index 8a9db605a..36ba6bed1 100644
--- a/openviking/storage/vectordb/store/store_manager.py
+++ b/openviking/storage/vectordb/store/store_manager.py
@@ -1,7 +1,7 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
import time
-from typing import List, Optional, Tuple, Union
+from typing import List, Optional, Tuple
from openviking.storage.vectordb.store.data import CandidateData, DeltaRecord
from openviking.storage.vectordb.store.local_store import create_store_engine_proxy
@@ -80,9 +80,7 @@ class StoreManager:
)
for bytes_data in bytes_list
]
- delta_list = [
- DeltaRecord(type=DeltaRecord.Type.UPSERT) for _ in range(len(cands_list))
- ]
+ delta_list = [DeltaRecord(type=DeltaRecord.Type.UPSERT) for _ in range(len(cands_list))]
for i, old_fields in enumerate(old_cands_fields_list):
delta_list[i].label = cands_list[i].label
delta_list[i].vector = cands_list[i].vector
@@ -154,9 +152,7 @@ class StoreManager:
)
for bytes_data in bytes_list
]
- delta_list = [
- DeltaRecord(type=DeltaRecord.Type.DELETE) for _ in range(len(label_list))
- ]
+ delta_list = [DeltaRecord(type=DeltaRecord.Type.DELETE) for _ in range(len(label_list))]
for i, data in enumerate(old_cands_fields_list):
delta_list[i].label = label_list[i]
delta_list[i].old_fields = data
@@ -181,9 +177,7 @@ class StoreManager:
self.storage.exec_sequence_batch_op(batch_op_list)
return delta_list
- def fetch_cands_data(
- self, label_list: List[int]
- ) -> List[Optional[CandidateData]]:
+ def fetch_cands_data(self, label_list: List[int]) -> List[Optional[CandidateData]]:
"""Fetch candidate data by labels.
Args:
@@ -272,13 +266,13 @@ class StoreManager:
# Optimize: Avoid full deserialization if only checking expire_ns_ts
# But we need label and fields later for DeltaRecord.
# Let's filter first by expire_ns_ts which is in CandidateData.
-
+
expired_cands_data: List[Tuple[int, str]] = []
-
+
for byte_data in cands_bytes_list:
if not byte_data:
continue
-
+
# Efficiently check expiration without full object creation if possible
# But CandidateData takes bytes_data in init, so we might as well use helper
expire_ts = CandidateData.bytes_row.deserialize_field(byte_data, "expire_ns_ts")
@@ -291,7 +285,7 @@ class StoreManager:
delta_list = [
DeltaRecord(type=DeltaRecord.Type.DELETE) for _ in range(len(expired_cands_data))
]
-
+
if expired_cands_data:
batch_op_list.append(
BatchOp(
@@ -304,7 +298,7 @@ class StoreManager:
for i, data in enumerate(expired_cands_data):
delta_list[i].label = data[0]
delta_list[i].old_fields = data[1]
-
+
base_ts = time.time_ns()
batch_op_list.append(
BatchOp(
@@ -324,6 +318,6 @@ class StoreManager:
["" for _ in range(len(ttl_kv_list))],
)
)
-
+
self.storage.exec_sequence_batch_op(batch_op_list)
return delta_list
diff --git a/openviking/storage/vectordb/utils/api_utils.py b/openviking/storage/vectordb/utils/api_utils.py
index c5924d337..58b31c052 100644
--- a/openviking/storage/vectordb/utils/api_utils.py
+++ b/openviking/storage/vectordb/utils/api_utils.py
@@ -2,6 +2,7 @@
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, Optional
+
def form_error(err: Any, code: int = 400, data: Optional[Dict] = None) -> tuple:
if data is None:
data = {}
diff --git a/openviking/storage/vectordb/utils/config_utils.py b/openviking/storage/vectordb/utils/config_utils.py
index 7457032fd..785ca8df4 100644
--- a/openviking/storage/vectordb/utils/config_utils.py
+++ b/openviking/storage/vectordb/utils/config_utils.py
@@ -3,6 +3,7 @@
import os
from typing import Any, Dict, Optional
+
def get_config_value(
config: Optional[Dict[str, Any]], config_key: str, env_var: str, default_value: Any
) -> Any:
diff --git a/openviking/storage/vectordb/utils/data_utils.py b/openviking/storage/vectordb/utils/data_utils.py
index 06a039526..cb8db6fe3 100644
--- a/openviking/storage/vectordb/utils/data_utils.py
+++ b/openviking/storage/vectordb/utils/data_utils.py
@@ -2,7 +2,8 @@
# SPDX-License-Identifier: Apache-2.0
import json
import logging
-from typing import Any, Dict, List, Union, Optional
+from typing import Dict, List, Union
+
def convert_dict(p: Union[str, Dict, List], no_exception: bool = False) -> Union[Dict, List]:
"""
diff --git a/openviking/storage/vectordb/utils/dict_utils.py b/openviking/storage/vectordb/utils/dict_utils.py
index fbf352d8a..c76b14a9d 100644
--- a/openviking/storage/vectordb/utils/dict_utils.py
+++ b/openviking/storage/vectordb/utils/dict_utils.py
@@ -132,8 +132,7 @@ def filter_dict_key_with_prefix(d: Dict[str, Any], prefix: str = "_") -> Dict[st
filtered[key] = filter_dict_key_with_prefix(value, prefix)
elif isinstance(value, list):
filtered[key] = [
- filter_dict_key_with_prefix(v, prefix) if isinstance(v, dict) else v
- for v in value
+ filter_dict_key_with_prefix(v, prefix) if isinstance(v, dict) else v for v in value
]
else:
filtered[key] = value
diff --git a/openviking/storage/vectordb/utils/file_utils.py b/openviking/storage/vectordb/utils/file_utils.py
index 206c4458d..95487fd98 100644
--- a/openviking/storage/vectordb/utils/file_utils.py
+++ b/openviking/storage/vectordb/utils/file_utils.py
@@ -2,6 +2,7 @@
# SPDX-License-Identifier: Apache-2.0
import logging
+
def touch_file(file_path: str) -> None:
try:
with open(file_path, "a"):
diff --git a/openviking/storage/vectordb/utils/id_generator.py b/openviking/storage/vectordb/utils/id_generator.py
index 52f4f4eb7..39ae5b581 100644
--- a/openviking/storage/vectordb/utils/id_generator.py
+++ b/openviking/storage/vectordb/utils/id_generator.py
@@ -1,37 +1,38 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
-import time
-import threading
import os
import random
+import threading
+import time
+
class SnowflakeGenerator:
"""
Distributed unique ID generator based on Twitter's Snowflake algorithm.
Generates 64-bit integers (int64/uint64 compatible).
-
+
Structure (64 bits):
- 1 bit: Unused (sign bit)
- 41 bits: Timestamp (milliseconds since epoch)
- 10 bits: Machine/Process ID (5 bits datacenter + 5 bits worker)
- 12 bits: Sequence number (per millisecond)
"""
-
+
# Constants
EPOCH = 1704067200000 # 2024-01-01 00:00:00 UTC
-
+
worker_id_bits = 5
datacenter_id_bits = 5
sequence_bits = 12
-
+
max_worker_id = -1 ^ (-1 << worker_id_bits)
max_datacenter_id = -1 ^ (-1 << datacenter_id_bits)
max_sequence = -1 ^ (-1 << sequence_bits)
-
+
worker_id_shift = sequence_bits
datacenter_id_shift = sequence_bits + worker_id_bits
timestamp_left_shift = sequence_bits + worker_id_bits + datacenter_id_bits
-
+
def __init__(self, worker_id: int = None, datacenter_id: int = None):
"""
Initialize the generator.
@@ -41,46 +42,48 @@ class SnowflakeGenerator:
# Use Process ID to distinguish processes on the same machine
# PID can be large, so we mask it to fit in worker_id_bits
worker_id = os.getpid() & self.max_worker_id
-
+
if datacenter_id is None:
# In a containerized environment, hostname usually changes.
# Using hash of hostname or a random number if not configured.
# For local single-node usage, random is acceptable initialization.
datacenter_id = random.randint(0, self.max_datacenter_id)
-
+
if worker_id > self.max_worker_id or worker_id < 0:
raise ValueError(f"worker_id must be between 0 and {self.max_worker_id}")
if datacenter_id > self.max_datacenter_id or datacenter_id < 0:
raise ValueError(f"datacenter_id must be between 0 and {self.max_datacenter_id}")
-
+
self.worker_id = worker_id
self.datacenter_id = datacenter_id
-
+
self.sequence = 0
self.last_timestamp = -1
self.lock = threading.Lock()
-
+
def _current_timestamp(self):
return int(time.time() * 1000)
-
+
def next_id(self) -> int:
"""
Generate the next unique ID.
"""
with self.lock:
timestamp = self._current_timestamp()
-
+
if timestamp < self.last_timestamp:
# Clock moved backwards, refuse to generate id
# Wait until clock catches up or throw error
offset = self.last_timestamp - timestamp
- if offset <= 5: # If offset is small, just wait
+ if offset <= 5: # If offset is small, just wait
time.sleep(offset / 1000.0 + 0.001)
timestamp = self._current_timestamp()
-
+
if timestamp < self.last_timestamp:
- raise Exception(f"Clock moved backwards. Refusing to generate id for {self.last_timestamp - timestamp} milliseconds")
-
+ raise Exception(
+ f"Clock moved backwards. Refusing to generate id for {self.last_timestamp - timestamp} milliseconds"
+ )
+
if self.last_timestamp == timestamp:
self.sequence = (self.sequence + 1) & self.max_sequence
if self.sequence == 0:
@@ -89,19 +92,23 @@ class SnowflakeGenerator:
timestamp = self._current_timestamp()
else:
self.sequence = 0
-
+
self.last_timestamp = timestamp
-
- new_id = ((timestamp - self.EPOCH) << self.timestamp_left_shift) | \
- (self.datacenter_id << self.datacenter_id_shift) | \
- (self.worker_id << self.worker_id_shift) | \
- self.sequence
-
+
+ new_id = (
+ ((timestamp - self.EPOCH) << self.timestamp_left_shift)
+ | (self.datacenter_id << self.datacenter_id_shift)
+ | (self.worker_id << self.worker_id_shift)
+ | self.sequence
+ )
+
return new_id
+
# Global instance
_default_generator = SnowflakeGenerator()
+
def generate_auto_id() -> int:
"""
Generate a globally unique 64-bit integer ID.
diff --git a/openviking/storage/vectordb/utils/logging_init.py b/openviking/storage/vectordb/utils/logging_init.py
index 049f062a4..75333e595 100644
--- a/openviking/storage/vectordb/utils/logging_init.py
+++ b/openviking/storage/vectordb/utils/logging_init.py
@@ -5,7 +5,6 @@
import sys
import threading
-
_cpp_logging_initialized = False
_cpp_logging_lock = threading.Lock()
diff --git a/openviking/storage/vectordb/utils/validation.py b/openviking/storage/vectordb/utils/validation.py
index 68bdd11c5..b42029824 100644
--- a/openviking/storage/vectordb/utils/validation.py
+++ b/openviking/storage/vectordb/utils/validation.py
@@ -1,29 +1,24 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
from enum import Enum
-from typing import Any, Dict, List, Literal, Optional, Union, Annotated
-import random
-
-from pydantic import (
- BaseModel,
- Field,
- ValidationError as PydanticValidationError,
- field_validator,
- model_validator,
- StringConstraints,
- ConfigDict
-)
-from openviking.utils.logger import default_logger as logger
+from typing import Annotated, Any, List, Literal, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+from pydantic import ValidationError as PydanticValidationError
+
from openviking.storage.vectordb.utils.id_generator import generate_auto_id
+
# Custom ValidationError for compatibility
class ValidationError(Exception):
def __init__(self, message: str, field_path: str = None):
self.field_path = field_path
super().__init__(message)
+
# --- Basic Validators ---
+
def validate_name_str(name: str) -> str:
if not name:
raise ValueError("name is empty or None")
@@ -33,13 +28,21 @@ def validate_name_str(name: str) -> str:
raise ValueError(f"name must start with a letter, got '{name[0]}'")
invalid_chars = [c for c in name if not (c.isalnum() or c == "_")]
if invalid_chars:
- raise ValueError(f"name can only contain letters, numbers and underscore, found invalid characters: {invalid_chars}")
+ raise ValueError(
+ f"name can only contain letters, numbers and underscore, found invalid characters: {invalid_chars}"
+ )
return name
-ValidName = Annotated[str, Field(min_length=1, max_length=128), field_validator('name', mode='before', check_fields=False)(validate_name_str)]
+
+ValidName = Annotated[
+ str,
+ Field(min_length=1, max_length=128),
+ field_validator("name", mode="before", check_fields=False)(validate_name_str),
+]
# --- Models ---
+
class FieldTypeEnum(str, Enum):
INT64 = "int64"
FLOAT32 = "float32"
@@ -56,9 +59,10 @@ class FieldTypeEnum(str, Enum):
DATE_TIME = "date_time"
GEO_POINT = "geo_point"
+
class DenseVectorize(BaseModel):
- model_config = ConfigDict(extra='forbid')
-
+ model_config = ConfigDict(extra="forbid")
+
ModelName: str
ModelVersion: Optional[str] = None
TextField: Optional[str] = None
@@ -67,66 +71,69 @@ class DenseVectorize(BaseModel):
Dim: Optional[int] = None
Dimension: Optional[int] = None
- @field_validator('TextField', 'ImageField', 'VideoField')
+ @field_validator("TextField", "ImageField", "VideoField")
@classmethod
def check_fields(cls, v):
- # We enforce presence logic in model_validator if needed,
+ # We enforce presence logic in model_validator if needed,
# but the original code had strict checks inside validate_dense_vectorize
# Original: "TextField" is required
return v
-
- @model_validator(mode='after')
+
+ @model_validator(mode="after")
def check_required(self):
if self.TextField is None and self.ImageField is None and self.VideoField is None:
- # Original logic: if "text_field" not in vectorize["dense"]
- # The old code strictly required "TextField" (or "text_field" in logic, but schema used "TextField")
- # Actually, old code: if "text_field" not in vectorize["dense"]: return False
- # But ALLOWED keys used "TextField". It seems there's a case sensitivity issue in old code or intended.
- # The old code check keys: 'ModelName', 'TextField' against ALLOWED_COLLECTION_DENSE_VECTORIZE_CHECK
- # Let's assume PascalCase as per ALLOWED dictionary keys.
- if not self.TextField:
- raise ValueError("vectorize dense must contain TextField")
+ # Original logic: if "text_field" not in vectorize["dense"]
+ # The old code strictly required "TextField" (or "text_field" in logic, but schema used "TextField")
+ # Actually, old code: if "text_field" not in vectorize["dense"]: return False
+ # But ALLOWED keys used "TextField". It seems there's a case sensitivity issue in old code or intended.
+ # The old code check keys: 'ModelName', 'TextField' against ALLOWED_COLLECTION_DENSE_VECTORIZE_CHECK
+ # Let's assume PascalCase as per ALLOWED dictionary keys.
+ if not self.TextField:
+ raise ValueError("vectorize dense must contain TextField")
return self
+
class SparseVectorize(BaseModel):
- model_config = ConfigDict(extra='forbid')
-
+ model_config = ConfigDict(extra="forbid")
+
ModelName: str
ModelVersion: Optional[str] = None
TextField: Optional[str] = None
+
class VectorizeConfig(BaseModel):
- model_config = ConfigDict(extra='forbid')
-
+ model_config = ConfigDict(extra="forbid")
+
Dense: Optional[DenseVectorize] = None
Sparse: Optional[SparseVectorize] = None
+
class CollectionField(BaseModel):
- model_config = ConfigDict(extra='forbid')
-
+ model_config = ConfigDict(extra="forbid")
+
FieldName: str
FieldType: FieldTypeEnum
Dim: Optional[int] = Field(None, ge=4, le=4096)
IsPrimaryKey: Optional[bool] = False
DefaultValue: Optional[Any] = None
-
+
# Internal fields
_FieldID: Optional[int] = None
- @field_validator('FieldName')
+ @field_validator("FieldName")
@classmethod
def validate_fieldname(cls, v):
return validate_name_str(v)
- @field_validator('Dim')
+ @field_validator("Dim")
@classmethod
def validate_dim(cls, v):
if v is not None:
if v % 4 != 0:
raise ValueError(f"dimension must be a multiple of 4, got {v}")
return v
-
- @model_validator(mode='after')
+
+ @model_validator(mode="after")
def validate_field_logic(cls, m):
if m.FieldType == FieldTypeEnum.VECTOR:
if m.Dim is None:
@@ -136,25 +143,27 @@ class CollectionField(BaseModel):
raise ValueError(f"primary key must be int64 or string, got '{m.FieldType}'")
return m
+
class CollectionMetaConfig(BaseModel):
- model_config = ConfigDict(extra='forbid')
-
+ model_config = ConfigDict(extra="forbid")
+
CollectionName: str
Fields: List[CollectionField]
ProjectName: Optional[str] = None
Description: Optional[str] = Field(None, max_length=65535)
Vectorize: Optional[VectorizeConfig] = None
-
+
# Internal fields
_FieldsCount: Optional[int] = None
- @field_validator('CollectionName', 'ProjectName')
+ @field_validator("CollectionName", "ProjectName")
@classmethod
def validate_names(cls, v):
- if v is None: return v
+ if v is None:
+ return v
return validate_name_str(v)
- @field_validator('Fields')
+ @field_validator("Fields")
@classmethod
def validate_fields_list(cls, fields):
names = set()
@@ -169,9 +178,10 @@ class CollectionMetaConfig(BaseModel):
has_pk = True
return fields
+
class VectorIndexConfig(BaseModel):
- model_config = ConfigDict(extra='forbid')
-
+ model_config = ConfigDict(extra="forbid")
+
IndexType: Literal["flat", "flat_hybrid", "FLAT", "FLAT_HYBRID"]
Distance: Optional[Literal["l2", "ip", "cosine", "L2", "IP", "COSINE"]] = None
Quant: Optional[Literal["int8", "float", "fix16", "pq", "INT8", "FLOAT", "FIX16", "PQ"]] = None
@@ -183,7 +193,7 @@ class VectorIndexConfig(BaseModel):
IndexWithSparseLogitAlpha: Optional[float] = None
EnableSparse: Optional[bool] = None
- @field_validator('IndexType', 'Distance', 'Quant', mode='before')
+ @field_validator("IndexType", "Distance", "Quant", mode="before")
@classmethod
def case_insensitive(cls, v):
if isinstance(v, str):
@@ -191,33 +201,34 @@ class VectorIndexConfig(BaseModel):
# Actually user input might be mixed. Pydantic validates against Literals.
# Let's normalize everything to lower case if the Literal allows it, or just pass through.
# The old code did .lower() checks.
- pass
+ pass
return v
-
- @field_validator('IndexType')
+
+ @field_validator("IndexType")
@classmethod
def validate_index_type(cls, v):
if v.lower() not in ["flat", "flat_hybrid"]:
- raise ValueError(f"invalid index type '{v}'")
+ raise ValueError(f"invalid index type '{v}'")
return v
- @field_validator('Distance')
+ @field_validator("Distance")
@classmethod
def validate_distance(cls, v):
if v and v.lower() not in ["l2", "ip", "cosine"]:
- raise ValueError(f"invalid distance type '{v}'")
+ raise ValueError(f"invalid distance type '{v}'")
return v
-
- @field_validator('Quant')
+
+ @field_validator("Quant")
@classmethod
def validate_quant(cls, v):
if v and v.lower() not in ["int8", "float", "fix16", "pq"]:
- raise ValueError(f"invalid quant type '{v}'")
+ raise ValueError(f"invalid quant type '{v}'")
return v
+
class IndexMetaConfig(BaseModel):
- model_config = ConfigDict(extra='forbid')
-
+ model_config = ConfigDict(extra="forbid")
+
IndexName: str
VectorIndex: VectorIndexConfig
ScalarIndex: Optional[List[str]] = None
@@ -225,15 +236,17 @@ class IndexMetaConfig(BaseModel):
ProjectName: Optional[str] = None
CollectionName: Optional[str] = None
- @field_validator('IndexName', 'ProjectName', 'CollectionName')
+ @field_validator("IndexName", "ProjectName", "CollectionName")
@classmethod
def validate_names(cls, v):
- if v is None: return v
+ if v is None:
+ return v
return validate_name_str(v)
+
class IndexMetaUpdateConfig(BaseModel):
- model_config = ConfigDict(extra='forbid')
-
+ model_config = ConfigDict(extra="forbid")
+
IndexName: Optional[str] = None
VectorIndex: Optional[VectorIndexConfig] = None
ScalarIndex: Optional[List[str]] = None
@@ -241,31 +254,35 @@ class IndexMetaUpdateConfig(BaseModel):
ProjectName: Optional[str] = None
CollectionName: Optional[str] = None
- @field_validator('IndexName', 'ProjectName', 'CollectionName')
+ @field_validator("IndexName", "ProjectName", "CollectionName")
@classmethod
def validate_names(cls, v):
- if v is None: return v
+ if v is None:
+ return v
return validate_name_str(v)
+
class CollectionMetaUpdateConfig(BaseModel):
- model_config = ConfigDict(extra='forbid')
-
+ model_config = ConfigDict(extra="forbid")
+
CollectionName: Optional[str] = None
Fields: Optional[List[CollectionField]] = None
ProjectName: Optional[str] = None
Description: Optional[str] = Field(None, max_length=65535)
Vectorize: Optional[VectorizeConfig] = None
-
- @field_validator('CollectionName', 'ProjectName')
+
+ @field_validator("CollectionName", "ProjectName")
@classmethod
def validate_names(cls, v):
- if v is None: return v
+ if v is None:
+ return v
return validate_name_str(v)
- @field_validator('Fields')
+ @field_validator("Fields")
@classmethod
def validate_fields_list(cls, fields):
- if fields is None: return fields
+ if fields is None:
+ return fields
names = set()
has_pk = False
for f in fields:
@@ -278,28 +295,32 @@ class CollectionMetaUpdateConfig(BaseModel):
has_pk = True
return fields
+
# --- Helper / Compatibility Functions ---
+
def _handle_validation_error(e: PydanticValidationError):
# Convert Pydantic ValidationError to our custom ValidationError string format
- # to maintain some resemblance of old error messages if needed,
+ # to maintain some resemblance of old error messages if needed,
# or just raise our custom exception.
msg = str(e)
# Extract first error for cleaner message
try:
err = e.errors()[0]
- field = ".".join(str(x) for x in err['loc'])
+ field = ".".join(str(x) for x in err["loc"])
msg = f"{err['msg']} (field: {field})"
except:
pass
raise ValidationError(msg)
+
def validate_collection_meta_data(meta_data: dict) -> None:
try:
CollectionMetaConfig.model_validate(meta_data)
except PydanticValidationError as e:
_handle_validation_error(e)
+
def is_valid_collection_meta_data(meta_data: dict) -> bool:
try:
validate_collection_meta_data(meta_data)
@@ -307,12 +328,14 @@ def is_valid_collection_meta_data(meta_data: dict) -> bool:
except ValidationError:
return False
+
def validate_collection_meta_data_for_update(meta_data: dict, field_meta_dict: dict = None) -> None:
try:
CollectionMetaUpdateConfig.model_validate(meta_data)
except PydanticValidationError as e:
_handle_validation_error(e)
+
def is_valid_collection_meta_data_for_update(meta_data: dict, field_meta_dict: dict = None) -> bool:
try:
validate_collection_meta_data_for_update(meta_data, field_meta_dict)
@@ -320,6 +343,7 @@ def is_valid_collection_meta_data_for_update(meta_data: dict, field_meta_dict: d
except ValidationError:
return False
+
def validate_index_meta_data(meta_data: dict, field_meta_dict: dict) -> None:
try:
model = IndexMetaConfig.model_validate(meta_data)
@@ -327,10 +351,13 @@ def validate_index_meta_data(meta_data: dict, field_meta_dict: dict) -> None:
if model.ScalarIndex:
unknown_fields = set(model.ScalarIndex) - set(field_meta_dict.keys())
if unknown_fields:
- raise ValidationError(f"scalar index contains unknown fields: {list(unknown_fields)}")
+ raise ValidationError(
+ f"scalar index contains unknown fields: {list(unknown_fields)}"
+ )
except PydanticValidationError as e:
_handle_validation_error(e)
+
def is_valid_index_meta_data(meta_data: dict, field_meta_dict: dict) -> bool:
try:
validate_index_meta_data(meta_data, field_meta_dict)
@@ -338,16 +365,20 @@ def is_valid_index_meta_data(meta_data: dict, field_meta_dict: dict) -> bool:
except ValidationError:
return False
+
def validate_index_meta_data_for_update(meta_data: dict, field_meta_dict: dict) -> None:
try:
model = IndexMetaUpdateConfig.model_validate(meta_data)
if model.ScalarIndex:
unknown_fields = set(model.ScalarIndex) - set(field_meta_dict.keys())
if unknown_fields:
- raise ValidationError(f"scalar index contains unknown fields: {list(unknown_fields)}")
+ raise ValidationError(
+ f"scalar index contains unknown fields: {list(unknown_fields)}"
+ )
except PydanticValidationError as e:
_handle_validation_error(e)
+
def is_valid_index_meta_data_for_update(meta_data: dict, field_meta_dict: dict) -> bool:
try:
validate_index_meta_data_for_update(meta_data, field_meta_dict)
@@ -355,6 +386,7 @@ def is_valid_index_meta_data_for_update(meta_data: dict, field_meta_dict: dict)
except ValidationError:
return False
+
def fix_collection_meta(meta_data: dict) -> dict:
# This logic mutates the input dict to add AUTO_ID if missing, etc.
# We can perform this on the dict directly or parse into model, modify, dump.
@@ -365,24 +397,27 @@ def fix_collection_meta(meta_data: dict) -> dict:
if item.get("IsPrimaryKey", False):
has_pk = True
break
-
+
if not has_pk:
- fields.append({
- "FieldName": "AUTO_ID",
- "FieldType": "int64",
- "IsPrimaryKey": True,
- })
-
+ fields.append(
+ {
+ "FieldName": "AUTO_ID",
+ "FieldType": "int64",
+ "IsPrimaryKey": True,
+ }
+ )
+
field_count = meta_data.get("_FieldsCount", 0)
for item in fields:
if "_FieldID" not in item:
item["_FieldID"] = field_count
field_count += 1
-
+
meta_data["_FieldsCount"] = field_count
meta_data["Fields"] = fields
return meta_data
+
# Data validation logic - kept mostly manual or lightweight as creating models for dynamic data row is expensive
REQUIRED_COLLECTION_FIELD_TYPE_CHECK = {
"int64": ([int], None, 0),
@@ -398,7 +433,7 @@ REQUIRED_COLLECTION_FIELD_TYPE_CHECK = {
"vector": (
[list],
# dim check is done elsewhere or we assume valid if it's a list of floats
- lambda l: all(isinstance(item, (int, float)) for item in l),
+ lambda l: all(isinstance(item, (int, float)) for item in l),
[],
),
"text": ([str], None, ""),
@@ -410,9 +445,12 @@ REQUIRED_COLLECTION_FIELD_TYPE_CHECK = {
"sparse_vector": ([dict], None, {}),
}
+
def validate_fields_data(field_data_dict: dict, field_meta_dict: dict) -> None:
if len(field_data_dict) > len(field_meta_dict):
- raise ValidationError(f"too many fields: got {len(field_data_dict)}, expected max {len(field_meta_dict)}")
+ raise ValidationError(
+ f"too many fields: got {len(field_data_dict)}, expected max {len(field_meta_dict)}"
+ )
for field_name, field_value in field_data_dict.items():
if field_name not in field_meta_dict:
@@ -420,29 +458,33 @@ def validate_fields_data(field_data_dict: dict, field_meta_dict: dict) -> None:
field_type = field_meta_dict[field_name]["FieldType"]
# Compatibility with enum if using Pydantic model for meta dict
- if hasattr(field_type, "value"):
+ if hasattr(field_type, "value"):
field_type = field_type.value
-
+
if field_type not in REQUIRED_COLLECTION_FIELD_TYPE_CHECK:
- # Should be caught by meta validation, but safety check
- continue
+ # Should be caught by meta validation, but safety check
+ continue
allowed_types, validator, _ = REQUIRED_COLLECTION_FIELD_TYPE_CHECK[field_type]
-
+
if type(field_value) not in allowed_types:
- raise ValidationError(f"field type mismatch for '{field_name}': expected {field_type}, got {type(field_value).__name__}")
-
+ raise ValidationError(
+ f"field type mismatch for '{field_name}': expected {field_type}, got {type(field_value).__name__}"
+ )
+
if validator and not validator(field_value):
- raise ValidationError(f"invalid value for field '{field_name}'")
+ raise ValidationError(f"invalid value for field '{field_name}'")
+
def is_valid_fields_data(field_data_dict: dict, field_meta_dict: dict) -> bool:
try:
validate_fields_data(field_data_dict, field_meta_dict)
return True
- except ValidationError as e:
+ except ValidationError:
# print(f"ValidationError {e}") # Reduce noise
return False
+
def fix_fields_data(field_data_dict: dict, field_meta_dict: dict) -> dict:
if len(field_data_dict) >= len(field_meta_dict):
return field_data_dict
@@ -457,7 +499,8 @@ def fix_fields_data(field_data_dict: dict, field_meta_dict: dict) -> dict:
field_type = field_meta.FieldType
default_val = field_meta.DefaultValue
- if hasattr(field_type, "value"): field_type = field_type.value
+ if hasattr(field_type, "value"):
+ field_type = field_type.value
if default_val is not None:
field_data_dict[field_name] = default_val
@@ -466,4 +509,3 @@ def fix_fields_data(field_data_dict: dict, field_meta_dict: dict) -> dict:
else:
field_data_dict[field_name] = REQUIRED_COLLECTION_FIELD_TYPE_CHECK[field_type][2]
return field_data_dict
-
diff --git a/openviking/storage/vectordb/vectorize/vectorizer.py b/openviking/storage/vectordb/vectorize/vectorizer.py
index e82aae62e..222728f65 100644
--- a/openviking/storage/vectordb/vectorize/vectorizer.py
+++ b/openviking/storage/vectordb/vectorize/vectorizer.py
@@ -2,6 +2,7 @@
# SPDX-License-Identifier: Apache-2.0
from typing import Any, Dict, List, Optional, Tuple, TypedDict
+
class DenseMeta(TypedDict, total=False):
ModelName: str
Version: str
@@ -10,14 +11,17 @@ class DenseMeta(TypedDict, total=False):
ImageField: str
VideoField: str
+
class SparseMeta(TypedDict, total=False):
ModelName: str
Version: str
+
class VectorizeMeta(TypedDict, total=False):
Dense: DenseMeta
Sparse: SparseMeta
+
class VectorizerAdapter:
"""Adapter for vectorizer to handle data vectorization.
@@ -63,7 +67,9 @@ class VectorizerAdapter:
"""
return self.dim
- def vectorize_raw_data(self, raw_data_list: List[Dict[str, Any]]) -> Tuple[List[List[float]], List[Dict[str, float]]]:
+ def vectorize_raw_data(
+ self, raw_data_list: List[Dict[str, Any]]
+ ) -> Tuple[List[List[float]], List[Dict[str, float]]]:
"""Vectorize a list of raw data items.
Args:
diff --git a/openviking/storage/vectordb/vectorize/volcengine_vectorizer.py b/openviking/storage/vectordb/vectorize/volcengine_vectorizer.py
index 7f5a2a015..62e3349f0 100644
--- a/openviking/storage/vectordb/vectorize/volcengine_vectorizer.py
+++ b/openviking/storage/vectordb/vectorize/volcengine_vectorizer.py
@@ -10,10 +10,7 @@ from volcengine.auth.SignerV4 import SignerV4
from volcengine.base.Request import Request
from volcengine.Credentials import Credentials
-from openviking.storage.vectordb.vectorize.base import (
- BaseVectorizer,
- VectorizeResult,
-)
+from openviking.storage.vectordb.vectorize.base import BaseVectorizer, VectorizeResult
class ClientForDataApi:
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index 5a253f75e..8525ad2c2 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -14,10 +14,10 @@ Responsibilities:
import asyncio
import json
-from pathlib import PurePath
from dataclasses import dataclass, field
from datetime import datetime
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+from pathlib import PurePath
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from openviking.storage.vikingdb_interface import VikingDBInterface
from openviking.utils.logger import get_logger
@@ -173,8 +173,8 @@ class VikingFS:
path = self._uri_to_path(uri)
result = await asyncio.to_thread(self.agfs.grep, path, pattern, True, case_insensitive)
# Change file to uri
- if result.get('matches', None) is None:
- result['matches'] = []
+ if result.get("matches", None) is None:
+ result["matches"] = []
for match in result.get("matches", []):
match["uri"] = self._path_to_uri(match.pop("file"))
return result
@@ -357,8 +357,11 @@ class VikingFS:
FindResult
"""
from openviking.retrieve import (
- ContextType, FindResult, HierarchicalRetriever,
- QueryPlan, TypedQuery,
+ ContextType,
+ FindResult,
+ HierarchicalRetriever,
+ QueryPlan,
+ TypedQuery,
)
from openviking.retrieve.intent_analyzer import IntentAnalyzer
@@ -423,7 +426,8 @@ class VikingFS:
async def _execute(tq: TypedQuery):
return await retriever.retrieve(
- tq, limit=limit,
+ tq,
+ limit=limit,
score_threshold=score_threshold,
metadata_filter=filter,
)
@@ -468,9 +472,7 @@ class VikingFS:
link_id = next(f"link_{i}" for i in range(1, 10000) if f"link_{i}" not in existing_ids)
- entries.append(
- RelationEntry(id=link_id, uris=uris, reason=reason)
- )
+ entries.append(RelationEntry(id=link_id, uris=uris, reason=reason))
await self._write_relation_table(from_path, entries)
logger.info(f"[VikingFS] Created link: {from_uri} -> {uris}")
@@ -544,6 +546,7 @@ class VikingFS:
def _infer_context_type(self, uri: str):
"""Infer context_type from URI."""
from openviking.retrieve import ContextType
+
if "/memories" in uri:
return ContextType.MEMORY
elif "/skills" in uri:
@@ -590,7 +593,9 @@ class VikingFS:
except Exception as e:
logger.warning(f"[VikingFS] Failed to delete {uri} from vector store: {e}")
- async def _update_vector_store_uris(self, uris: List[str], old_base: str, new_base: str) -> None:
+ async def _update_vector_store_uris(
+ self, uris: List[str], old_base: str, new_base: str
+ ) -> None:
"""Update URIs in vector store (when moving files).
Preserves vector data, only updates uri and parent_uri fields, no need to regenerate embeddings.
@@ -619,12 +624,18 @@ class VikingFS:
new_uri = uri.replace(old_base_uri, new_base_uri, 1)
old_parent_uri = record.get("parent_uri", "")
- new_parent_uri = old_parent_uri.replace(old_base_uri, new_base_uri, 1) if old_parent_uri else ""
+ new_parent_uri = (
+ old_parent_uri.replace(old_base_uri, new_base_uri, 1) if old_parent_uri else ""
+ )
- await storage.update("context", record_id, {
- "uri": new_uri,
- "parent_uri": new_parent_uri,
- })
+ await storage.update(
+ "context",
+ record_id,
+ {
+ "uri": new_uri,
+ "parent_uri": new_parent_uri,
+ },
+ )
logger.info(f"[VikingFS] Updated URI: {uri} -> {new_uri}")
except Exception as e:
logger.warning(f"[VikingFS] Failed to update {uri} in vector store: {e}")
@@ -656,12 +667,10 @@ class VikingFS:
table_path = f"{dir_path}/.relations.json"
try:
content = await asyncio.to_thread(self.agfs.read, table_path)
- data = json.loads(
- content.decode("utf-8") if isinstance(content, bytes) else content
- )
+ data = json.loads(content.decode("utf-8") if isinstance(content, bytes) else content)
except FileNotFoundError:
return []
- except Exception as e:
+ except Exception:
# logger.warning(f"[VikingFS] Failed to read relation table {table_path}: {e}")
return []
@@ -673,8 +682,8 @@ class VikingFS:
entries.append(RelationEntry.from_dict(entry_data))
elif isinstance(data, dict):
# Old format: nested {namespace: {user: [entries]}}
- for namespace, user_dict in data.items():
- for user, entry_list in user_dict.items():
+ for _namespace, user_dict in data.items():
+ for _user, entry_list in user_dict.items():
for entry_data in entry_list:
entries.append(RelationEntry.from_dict(entry_data))
return entries
@@ -811,6 +820,7 @@ class VikingFS:
def create_temp_uri(self) -> str:
"""Create temp directory URI."""
import uuid
+
temp_id = uuid.uuid4().hex[:8]
return f"viking://temp/{temp_id}"
diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.py
index f67a3e200..87b300bc0 100644
--- a/openviking/storage/viking_vector_index_backend.py
+++ b/openviking/storage/viking_vector_index_backend.py
@@ -9,15 +9,12 @@ Supports both in-memory and local persistent storage modes.
import uuid
from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple
-from openviking.storage.vikingdb_interface import (
- CollectionNotFoundError,
- VikingDBInterface,
-)
from openviking.storage.vectordb.collection.collection import Collection
from openviking.storage.vectordb.collection.result import FetchDataInCollectionResult
from openviking.storage.vectordb.utils.logging_init import init_cpp_logging
+from openviking.storage.vikingdb_interface import CollectionNotFoundError, VikingDBInterface
from openviking.utils import get_logger
from openviking.utils.config.vectordb_config import VectorDBBackendConfig
@@ -82,25 +79,23 @@ class VikingVectorIndexBackend(VikingDBInterface):
self.vector_dim = config.vector_dim
self.distance_metric = config.distance_metric
-
+
if config.backend == "volcengine":
if not (
- config.volcengine
+ config.volcengine
and config.volcengine.ak
and config.volcengine.sk
and config.volcengine.region
):
- raise ValueError(
- "Volcengine backend requires AK, SK, and Region configuration"
- )
-
+ raise ValueError("Volcengine backend requires AK, SK, and Region configuration")
+
# Volcengine VikingDB mode
self._mode = config.backend
# Convert lowercase keys to uppercase for consistency with volcengine_collection
volc_config = {
- "AK": config.volcengine.ak,
- "SK": config.volcengine.sk,
- "Region": config.volcengine.region,
+ "AK": config.volcengine.ak,
+ "SK": config.volcengine.sk,
+ "Region": config.volcengine.region,
}
from openviking.storage.vectordb.project.volcengine_project import (
@@ -239,7 +234,11 @@ class VikingVectorIndexBackend(VikingDBInterface):
is_primary_key = field.get("IsPrimaryKey", False)
# Index all non-vector, non-primary-key, and non-date_time fields by default
# Volcengine VikingDB doesn't support indexing date_time fields
- if field_name and field_type not in ("vector", "sparse_vector", "date_time") and not is_primary_key:
+ if (
+ field_name
+ and field_type not in ("vector", "sparse_vector", "date_time")
+ and not is_primary_key
+ ):
scalar_index_fields.append(field_name)
# Create default index for the collection
@@ -308,9 +307,7 @@ class VikingVectorIndexBackend(VikingDBInterface):
if not self.project.has_collection(name):
return None
- collection = self.project.get_collection(name)
config = self._collection_configs.get(name, {})
- meta_data = self._get_meta_data(name, collection)
return {
"name": name,
@@ -354,7 +351,7 @@ class VikingVectorIndexBackend(VikingDBInterface):
new_data[k] = data[k]
try:
- result = coll.upsert_data([new_data])
+ coll.upsert_data([new_data])
return record_id
except Exception as e:
logger.error(f"Error inserting record: {e}")
@@ -445,22 +442,22 @@ class VikingVectorIndexBackend(VikingDBInterface):
result = coll.search_by_random(
index_name=self.DEFAULT_INDEX_NAME,
limit=10,
- filters={"op": "must", "field": "uri", "conds": [uri]}
+ filters={"op": "must", "field": "uri", "conds": [uri]},
)
records = []
for item in result.data:
record = dict(item.fields) if item.fields else {}
record["id"] = item.id
records.append(record)
- if (len(records) > 0):
+ if len(records) > 0:
raise ValueError(f"Duplicate records found for URI: {uri}")
- if (len(records) == 0):
+ if len(records) == 0:
raise ValueError(f"Record not found for URI: {uri}")
return records[0]
except Exception as e:
logger.error(f"Error fetching record by URI '{uri}': {e}")
return None
-
+
async def exists(self, collection: str, id: str) -> bool:
"""Check if a record exists."""
try:
@@ -842,7 +839,7 @@ class VikingVectorIndexBackend(VikingDBInterface):
"""Check if storage backend is healthy and accessible."""
try:
# Simple check: verify we can access the project
- collections = self.project.list_collections()
+ self.project.list_collections()
return True
except Exception:
return False
@@ -858,9 +855,7 @@ class VikingVectorIndexBackend(VikingDBInterface):
try:
coll = self._get_collection(collection_name)
result = coll.aggregate_data(
- index_name=self.DEFAULT_INDEX_NAME,
- op="count",
- filters=None
+ index_name=self.DEFAULT_INDEX_NAME, op="count", filters=None
)
total_records += result.agg.get("_total", 0)
except Exception as e:
diff --git a/openviking/storage/vikingdb_manager.py b/openviking/storage/vikingdb_manager.py
index 8ecfb8896..af38c9e6f 100644
--- a/openviking/storage/vikingdb_manager.py
+++ b/openviking/storage/vikingdb_manager.py
@@ -4,13 +4,13 @@
VikingDB Manager class that extends VikingVectorIndexBackend with queue management functionality.
"""
-from typing import Any, Dict, List, Optional, Union
-from pathlib import Path
-from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend
-from openviking.storage.queuefs.queue_manager import init_queue_manager
+from typing import Optional
+
from openviking.storage.collection_schemas import TextEmbeddingHandler
from openviking.storage.queuefs.embedding_msg import EmbeddingMsg
from openviking.storage.queuefs.embedding_queue import EmbeddingQueue
+from openviking.storage.queuefs.queue_manager import init_queue_manager
+from openviking.storage.viking_vector_index_backend import VikingVectorIndexBackend
from openviking.utils import get_logger
from openviking.utils.config.agfs_config import AGFSConfig
from openviking.utils.config.vectordb_config import VectorDBBackendConfig
@@ -86,12 +86,12 @@ class VikingDBManager(VikingVectorIndexBackend):
self._embedding_handler = TextEmbeddingHandler(self)
# Get embedding queue with the handler, allow creation if not exists
- embedding_queue = self._queue_manager.get_queue(
+ self._queue_manager.get_queue(
self._queue_manager.EMBEDDING,
dequeue_handler=self._embedding_handler,
allow_create=True,
)
- logger.info(f"Embedding queue initialized with TextEmbeddingHandler")
+ logger.info("Embedding queue initialized with TextEmbeddingHandler")
def _init_semantic_queue(self):
"""Initialize semantic queue with SemanticProcessor, semantic queue is used to get abstract and summary of context data."""
@@ -105,12 +105,12 @@ class VikingDBManager(VikingVectorIndexBackend):
self._semantic_processor = SemanticProcessor()
# Get semantic queue with the handler, allow creation if not exists
- semantic_queue = self._queue_manager.get_queue(
+ self._queue_manager.get_queue(
self._queue_manager.SEMANTIC,
dequeue_handler=self._semantic_processor,
allow_create=True,
)
- logger.info(f"Semantic queue initialized with SemanticProcessor")
+ logger.info("Semantic queue initialized with SemanticProcessor")
async def close(self) -> None:
"""Close storage connection and release resources, including queue manager."""
diff --git a/openviking/sync_client.py b/openviking/sync_client.py
index 05d4316ce..fbe61b70f 100644
--- a/openviking/sync_client.py
+++ b/openviking/sync_client.py
@@ -5,10 +5,9 @@ Synchronous OpenViking client implementation.
"""
import asyncio
-from typing import Any, Dict, List, Optional, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
if TYPE_CHECKING:
- from openviking.retrieve import QueryResult
from openviking.session import Session
from openviking.async_client import AsyncOpenViking
@@ -66,9 +65,17 @@ class SyncOpenViking:
filter: Optional[Dict] = None,
):
"""Execute complex retrieval (intent analysis, hierarchical retrieval)."""
- return asyncio.run(self._async_client.search(query, target_uri, session, limit, score_threshold, filter))
+ return asyncio.run(
+ self._async_client.search(query, target_uri, session, limit, score_threshold, filter)
+ )
- def find(self, query: str, target_uri: str = "", limit: int = 10, score_threshold: Optional[float] = None):
+ def find(
+ self,
+ query: str,
+ target_uri: str = "",
+ limit: int = 10,
+ score_threshold: Optional[float] = None,
+ ):
"""Quick retrieval"""
return asyncio.run(self._async_client.find(query, target_uri, limit, score_threshold))
@@ -140,7 +147,7 @@ class SyncOpenViking:
def mv(self, from_uri: str, to_uri: str) -> None:
"""Move resource"""
return asyncio.run(self._async_client.mv(from_uri, to_uri))
-
+
def tree(self, uri: str) -> Dict:
"""Get directory tree"""
return asyncio.run(self._async_client.tree(uri))
diff --git a/openviking/utils/config/__init__.py b/openviking/utils/config/__init__.py
index bac6cf272..e6a811413 100644
--- a/openviking/utils/config/__init__.py
+++ b/openviking/utils/config/__init__.py
@@ -1,33 +1,32 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
+from .agfs_config import AGFSConfig
from .embedding_config import EmbeddingConfig
from .open_viking_config import (
OpenVikingConfig,
OpenVikingConfigSingleton,
get_openviking_config,
- set_openviking_config,
is_valid_openviking_config,
+ set_openviking_config,
)
-from .rerank_config import RerankConfig
-from .storage_config import StorageConfig
-from .agfs_config import AGFSConfig
-from .vectordb_config import VectorDBBackendConfig
-from .vlm_config import VLMConfig
from .parser_config import (
- ParserConfig,
- PDFConfig,
+ PARSER_CONFIG_REGISTRY,
+ AudioConfig,
CodeConfig,
+ HTMLConfig,
ImageConfig,
- AudioConfig,
- VideoConfig,
MarkdownConfig,
- HTMLConfig,
+ ParserConfig,
+ PDFConfig,
TextConfig,
+ VideoConfig,
get_parser_config,
load_parser_configs_from_dict,
- PARSER_CONFIG_REGISTRY,
)
-
+from .rerank_config import RerankConfig
+from .storage_config import StorageConfig
+from .vectordb_config import VectorDBBackendConfig
+from .vlm_config import VLMConfig
__all__ = [
"AGFSConfig",
diff --git a/openviking/utils/config/agfs_config.py b/openviking/utils/config/agfs_config.py
index 7d3f12f76..166f1ae40 100644
--- a/openviking/utils/config/agfs_config.py
+++ b/openviking/utils/config/agfs_config.py
@@ -14,7 +14,9 @@ class AGFSConfig(BaseModel):
log_level: str = Field(default="warn", description="AGFS log level")
- url: Optional[str] = Field(default="http://localhost:8080", description="AGFS service URL for service mode")
+ url: Optional[str] = Field(
+ default="http://localhost:8080", description="AGFS service URL for service mode"
+ )
backend: str = Field(
default="local", description="AGFS storage backend: 'local' | 's3' | 'memory'"
@@ -33,7 +35,7 @@ class AGFSConfig(BaseModel):
s3_secret_key: Optional[str] = Field(default=None, description="S3 secret key")
- @model_validator(mode='after')
+ @model_validator(mode="after")
def validate_config(self):
"""Validate configuration completeness and consistency"""
if self.backend not in ["local", "s3", "memory"]:
diff --git a/openviking/utils/config/embedding_config.py b/openviking/utils/config/embedding_config.py
index b43be6d52..ae3ef8c7b 100644
--- a/openviking/utils/config/embedding_config.py
+++ b/openviking/utils/config/embedding_config.py
@@ -1,39 +1,45 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
import os
-from typing import List, Optional, Union, Dict, Any
-import openai
-import volcenginesdkarkruntime
+from typing import Any, Optional
+
from pydantic import BaseModel, Field, model_validator
+
class EmbeddingModelConfig(BaseModel):
"""Configuration for a specific embedding model"""
+
model: Optional[str] = Field(default=None, description="Model name")
api_key: Optional[str] = Field(default=None, description="API key")
api_base: Optional[str] = Field(default=None, description="API base URL")
dimension: Optional[int] = Field(default=None, description="Embedding dimension")
batch_size: int = Field(default=32, description="Batch size for embedding generation")
input: str = Field(default="multimodal", description="Input type: 'text' or 'multimodal'")
- provider: Optional[str] = Field(default="volcengine", description="Provider type: 'openai', 'volcengine', 'vikingdb'")
- backend: Optional[str] = Field(default="volcengine", description="Backend type (Deprecated, use 'provider' instead): 'openai', 'volcengine', 'vikingdb'")
+ provider: Optional[str] = Field(
+ default="volcengine", description="Provider type: 'openai', 'volcengine', 'vikingdb'"
+ )
+ backend: Optional[str] = Field(
+ default="volcengine",
+ description="Backend type (Deprecated, use 'provider' instead): 'openai', 'volcengine', 'vikingdb'",
+ )
version: Optional[str] = Field(default=None, description="Model version")
ak: Optional[str] = Field(default=None, description="Access Key ID for VikingDB API")
sk: Optional[str] = Field(default=None, description="Access Key Secretfor VikingDB API")
region: Optional[str] = Field(default=None, description="Region for VikingDB API")
host: Optional[str] = Field(default=None, description="Host for VikingDB API")
- @model_validator(mode='before')
+ @model_validator(mode="before")
@classmethod
def sync_provider_backend(cls, data: Any) -> Any:
if isinstance(data, dict):
- provider = data.get('provider')
- backend = data.get('backend')
-
+ provider = data.get("provider")
+ backend = data.get("backend")
+
if backend is not None and provider is None:
- data['provider'] = backend
+ data["provider"] = backend
return data
- @model_validator(mode='after')
+ @model_validator(mode="after")
def validate_config(self):
"""Validate configuration completeness and consistency"""
if self.backend and not self.provider:
@@ -101,14 +107,23 @@ class EmbeddingConfig(BaseModel):
config_sections = [
("dense", "OPENVIKING_EMBEDDING_DENSE_"),
("sparse", "OPENVIKING_EMBEDDING_SPARSE_"),
- ("hybrid", "OPENVIKING_EMBEDDING_HYBRID_")
+ ("hybrid", "OPENVIKING_EMBEDDING_HYBRID_"),
]
env_fields = {
- "model": str, "api_key": str, "api_base": str,
- "dimension": int, "batch_size": int,
- "input": str, "provider": str, "backend": str, "version": str,
- "ak": str, "sk": str, "region": str, "host": str
+ "model": str,
+ "api_key": str,
+ "api_base": str,
+ "dimension": int,
+ "batch_size": int,
+ "input": str,
+ "provider": str,
+ "backend": str,
+ "version": str,
+ "ak": str,
+ "sk": str,
+ "region": str,
+ "host": str,
}
for section, prefix in config_sections:
@@ -133,7 +148,7 @@ class EmbeddingConfig(BaseModel):
return data
- @model_validator(mode='after')
+ @model_validator(mode="after")
def validate_config(self):
"""Validate configuration completeness and consistency"""
if not self.dense and not self.sparse and not self.hybrid:
@@ -158,89 +173,89 @@ class EmbeddingConfig(BaseModel):
"""
from openviking.models.embedder import (
OpenAIDenseEmbedder,
- VolcengineDenseEmbedder,
- VolcengineSparseEmbedder,
- VolcengineHybridEmbedder,
VikingDBDenseEmbedder,
+ VikingDBHybridEmbedder,
VikingDBSparseEmbedder,
- VikingDBHybridEmbedder
+ VolcengineDenseEmbedder,
+ VolcengineHybridEmbedder,
+ VolcengineSparseEmbedder,
)
# Factory registry: (provider, type) -> (embedder_class, param_builder)
factory_registry = {
- ('openai', 'dense'): (
+ ("openai", "dense"): (
OpenAIDenseEmbedder,
lambda cfg: {
- 'model_name': cfg.model,
- 'api_key': cfg.api_key,
- 'api_base': cfg.api_base,
- 'dimension': cfg.dimension
- }
+ "model_name": cfg.model,
+ "api_key": cfg.api_key,
+ "api_base": cfg.api_base,
+ "dimension": cfg.dimension,
+ },
),
- ('volcengine', 'dense'): (
+ ("volcengine", "dense"): (
VolcengineDenseEmbedder,
lambda cfg: {
- 'model_name': cfg.model,
- 'api_key': cfg.api_key,
- 'api_base': cfg.api_base,
- 'dimension': cfg.dimension,
- 'input_type': cfg.input
- }
+ "model_name": cfg.model,
+ "api_key": cfg.api_key,
+ "api_base": cfg.api_base,
+ "dimension": cfg.dimension,
+ "input_type": cfg.input,
+ },
),
- ('volcengine', 'sparse'): (
+ ("volcengine", "sparse"): (
VolcengineSparseEmbedder,
lambda cfg: {
- 'model_name': cfg.model,
- 'api_key': cfg.api_key,
- 'api_base': cfg.api_base
- }
+ "model_name": cfg.model,
+ "api_key": cfg.api_key,
+ "api_base": cfg.api_base,
+ },
),
- ('volcengine', 'hybrid'): (
+ ("volcengine", "hybrid"): (
VolcengineHybridEmbedder,
lambda cfg: {
- 'model_name': cfg.model,
- 'api_key': cfg.api_key,
- 'api_base': cfg.api_base,
- 'dimension': cfg.dimension,
- 'input_type': cfg.input
- }
+ "model_name": cfg.model,
+ "api_key": cfg.api_key,
+ "api_base": cfg.api_base,
+ "dimension": cfg.dimension,
+ "input_type": cfg.input,
+ },
),
- ('vikingdb', 'dense'): (
+ ("vikingdb", "dense"): (
VikingDBDenseEmbedder,
lambda cfg: {
- 'model_name': cfg.model,
- 'model_version': cfg.version,
- 'ak': cfg.ak,
- 'sk': cfg.sk,
- 'region': cfg.region,
- 'host': cfg.host,
- 'dimension': cfg.dimension,
- 'input_type': cfg.input
- }
+ "model_name": cfg.model,
+ "model_version": cfg.version,
+ "ak": cfg.ak,
+ "sk": cfg.sk,
+ "region": cfg.region,
+ "host": cfg.host,
+ "dimension": cfg.dimension,
+ "input_type": cfg.input,
+ },
),
- ('vikingdb', 'sparse'): (
+ ("vikingdb", "sparse"): (
VikingDBSparseEmbedder,
lambda cfg: {
- 'model_name': cfg.model,
- 'model_version': cfg.version,
- 'ak': cfg.ak,
- 'sk': cfg.sk,
- 'region': cfg.region,
- 'host': cfg.host
- }
+ "model_name": cfg.model,
+ "model_version": cfg.version,
+ "ak": cfg.ak,
+ "sk": cfg.sk,
+ "region": cfg.region,
+ "host": cfg.host,
+ },
),
- ('vikingdb', 'hybrid'): (
+ ("vikingdb", "hybrid"): (
VikingDBHybridEmbedder,
lambda cfg: {
- 'model_name': cfg.model,
- 'model_version': cfg.version,
- 'ak': cfg.ak,
- 'sk': cfg.sk,
- 'region': cfg.region,
- 'host': cfg.host,
- 'dimension': cfg.dimension,
- 'input_type': cfg.input
- }
+ "model_name": cfg.model,
+ "model_version": cfg.version,
+ "ak": cfg.ak,
+ "sk": cfg.sk,
+ "region": cfg.region,
+ "host": cfg.host,
+ "dimension": cfg.dimension,
+ "input_type": cfg.input,
+ },
),
}
@@ -267,15 +282,17 @@ class EmbeddingConfig(BaseModel):
from openviking.models.embedder import CompositeHybridEmbedder
if self.hybrid:
- return self._create_embedder(self.hybrid.provider.lower(), 'hybrid', self.hybrid)
+ return self._create_embedder(self.hybrid.provider.lower(), "hybrid", self.hybrid)
if self.dense and self.sparse:
- dense_embedder = self._create_embedder(self.dense.provider.lower(), 'dense', self.dense)
- sparse_embedder = self._create_embedder(self.sparse.provider.lower(), 'sparse', self.sparse)
+ dense_embedder = self._create_embedder(self.dense.provider.lower(), "dense", self.dense)
+ sparse_embedder = self._create_embedder(
+ self.sparse.provider.lower(), "sparse", self.sparse
+ )
return CompositeHybridEmbedder(dense_embedder, sparse_embedder)
if self.dense:
- return self._create_embedder(self.dense.provider.lower(), 'dense', self.dense)
+ return self._create_embedder(self.dense.provider.lower(), "dense", self.dense)
raise ValueError("No embedding configuration found (dense, sparse, or hybrid)")
@@ -291,4 +308,3 @@ class EmbeddingConfig(BaseModel):
if self.dense:
return self.dense.dimension or 2048
return 2048
-
diff --git a/openviking/utils/config/open_viking_config.py b/openviking/utils/config/open_viking_config.py
index 5b0aea451..fe5c17051 100644
--- a/openviking/utils/config/open_viking_config.py
+++ b/openviking/utils/config/open_viking_config.py
@@ -1,29 +1,27 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
import json
-import logging
import os
from pathlib import Path
-from typing import Any, Dict, Optional
from threading import Lock
+from typing import Any, Dict, Optional
from pydantic import BaseModel, Field
from .embedding_config import EmbeddingConfig
-from .rerank_config import RerankConfig
-from .storage_config import StorageConfig
-from .vlm_config import VLMConfig
from .parser_config import (
- PDFConfig,
+ AudioConfig,
CodeConfig,
+ HTMLConfig,
ImageConfig,
- AudioConfig,
- VideoConfig,
MarkdownConfig,
- HTMLConfig,
+ PDFConfig,
TextConfig,
- load_parser_configs_from_dict,
+ VideoConfig,
)
+from .rerank_config import RerankConfig
+from .storage_config import StorageConfig
+from .vlm_config import VLMConfig
class OpenVikingConfig(BaseModel):
@@ -246,7 +244,9 @@ def is_valid_openviking_config(config: OpenVikingConfig) -> bool:
)
if errors:
- error_message = "Invalid OpenViking configuration:\n" + "\n".join(f" - {e}" for e in errors)
+ error_message = "Invalid OpenViking configuration:\n" + "\n".join(
+ f" - {e}" for e in errors
+ )
raise ValueError(error_message)
return True
diff --git a/openviking/utils/config/parser_config.py b/openviking/utils/config/parser_config.py
index 3e3ba0343..fbfb794d9 100644
--- a/openviking/utils/config/parser_config.py
+++ b/openviking/utils/config/parser_config.py
@@ -8,10 +8,10 @@ scattered across different modules. All configurations inherit from ParserConfig
and can be loaded from ov.conf files.
"""
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, Optional, Union
@dataclass
@@ -111,11 +111,11 @@ class ParserConfig:
value = os.environ[env_var]
# Type conversion
field_type = cls.__dataclass_fields__[field_name].type
- if field_type == bool:
+ if field_type is bool:
data[field_name] = value.lower() in ("true", "1", "yes")
- elif field_type == int:
+ elif field_type is int:
data[field_name] = int(value)
- elif field_type == float:
+ elif field_type is float:
data[field_name] = float(value)
else:
data[field_name] = value
diff --git a/openviking/utils/config/storage_config.py b/openviking/utils/config/storage_config.py
index cfd6d8731..c9661012c 100644
--- a/openviking/utils/config/storage_config.py
+++ b/openviking/utils/config/storage_config.py
@@ -1,17 +1,17 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
-from typing import Any, Dict, Optional
+from typing import Any, Dict
from pydantic import BaseModel, Field
from .agfs_config import AGFSConfig
from .vectordb_config import VectorDBBackendConfig
+
class StorageConfig(BaseModel):
"""Configuration for storage backend."""
- agfs: AGFSConfig = Field(
- default_factory=lambda: AGFSConfig(), description="AGFS configuration"
- )
+
+ agfs: AGFSConfig = Field(default_factory=lambda: AGFSConfig(), description="AGFS configuration")
vectordb: VectorDBBackendConfig = Field(
default_factory=lambda: VectorDBBackendConfig(),
diff --git a/openviking/utils/config/vectordb_config.py b/openviking/utils/config/vectordb_config.py
index 85b7587ec..dc32c7e42 100644
--- a/openviking/utils/config/vectordb_config.py
+++ b/openviking/utils/config/vectordb_config.py
@@ -1,11 +1,13 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
-from typing import Any, Dict, Optional
+from typing import Optional
from pydantic import BaseModel, Field, model_validator
+
class VolcengineConfig(BaseModel):
"""Configuration for Volcengine VikingDB."""
+
ak: Optional[str] = Field(default=None, description="Volcengine Access Key")
sk: Optional[str] = Field(default=None, description="Volcengine Secret Key")
region: Optional[str] = Field(
@@ -13,6 +15,7 @@ class VolcengineConfig(BaseModel):
)
host: Optional[str] = Field(default=None, description="Volcengine VikingDB host (optional)")
+
class VectorDBBackendConfig(BaseModel):
"""
Configuration for VectorDB backend.
@@ -20,6 +23,7 @@ class VectorDBBackendConfig(BaseModel):
This configuration class consolidates all settings related to the VectorDB backend,
including type, connection details, and backend-specific parameters.
"""
+
backend: str = Field(
default="local",
description="VectorDB backend type: 'local' (file-based), 'http' (remote service), or 'volcengine' (VikingDB)",
@@ -49,7 +53,7 @@ class VectorDBBackendConfig(BaseModel):
description="Volcengine VikingDB configuration for 'volcengine' type",
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def validate_config(self):
"""Validate configuration completeness and consistency"""
if self.backend not in ["local", "http", "volcengine"]:
@@ -71,4 +75,4 @@ class VectorDBBackendConfig(BaseModel):
if not self.volcengine.region:
raise ValueError("VectorDB volcengine backend requires 'region' to be set")
- return self
\ No newline at end of file
+ return self
diff --git a/openviking/utils/config/vlm_config.py b/openviking/utils/config/vlm_config.py
index 039c4f945..6511b9e68 100644
--- a/openviking/utils/config/vlm_config.py
+++ b/openviking/utils/config/vlm_config.py
@@ -1,11 +1,11 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
import os
-from typing import Any, Dict, Literal, Optional
+from typing import Any, Literal, Optional
from pydantic import BaseModel, Field, model_validator
-from openviking.models.vlm import VLMFactory, VLMBase
+from openviking.models.vlm import VLMBase, VLMFactory
class VLMConfig(BaseModel):
@@ -28,15 +28,15 @@ class VLMConfig(BaseModel):
class Config:
arbitrary_types_allowed = True
- @model_validator(mode='before')
+ @model_validator(mode="before")
@classmethod
def sync_provider_backend(cls, data: Any) -> Any:
if isinstance(data, dict):
- provider = data.get('provider')
- backend = data.get('backend')
-
+ provider = data.get("provider")
+ backend = data.get("backend")
+
if backend is not None and provider is None:
- data['provider'] = backend
+ data["provider"] = backend
return data
@model_validator(mode="before")
@@ -58,7 +58,7 @@ class VLMConfig(BaseModel):
data[field] = env_val
return data
- @model_validator(mode='after')
+ @model_validator(mode="after")
def validate_config(self):
"""Validate configuration completeness and consistency"""
if self.backend and not self.provider:
diff --git a/openviking/utils/llm.py b/openviking/utils/llm.py
index 5919afca5..d865a9d8e 100644
--- a/openviking/utils/llm.py
+++ b/openviking/utils/llm.py
@@ -10,16 +10,16 @@ import json
import re
from typing import Any, Dict, Optional, Type, TypeVar
+import json_repair
from pydantic import BaseModel
from openviking.utils.logger import get_logger
-import json_repair
logger = get_logger(__name__)
T = TypeVar("T", bound=BaseModel)
-
+
def parse_json_from_response(response: str) -> Optional[Any]:
"""
Parse JSON object from LLM text response.
@@ -78,6 +78,7 @@ def parse_json_from_response(response: str) -> Optional[Any]:
return None
+
def _fix_json_quotes(json_str: str) -> str:
"""
Fix quote issues in JSON string
@@ -106,6 +107,7 @@ def _fix_json_quotes(json_str: str) -> str:
except:
return json_str
+
def parse_json_to_model(response: str, model_class: Type[T]) -> Optional[T]:
"""
Parse JSON response into a Pydantic model.
@@ -170,6 +172,7 @@ class StructuredLLM:
def _get_vlm(self):
"""Get VLM singleton."""
from openviking.utils.config import get_openviking_config
+
return get_openviking_config().vlm
def complete_json(
diff --git a/openviking/utils/media_processor.py b/openviking/utils/media_processor.py
index 9efdc5560..39de9ae2a 100644
--- a/openviking/utils/media_processor.py
+++ b/openviking/utils/media_processor.py
@@ -31,6 +31,7 @@ class UnifiedResourceProcessor:
def _get_vlm_processor(self) -> Optional["VLMProcessor"]:
if self._vlm_processor is None:
from openviking.parse.vlm import VLMProcessor
+
self._vlm_processor = VLMProcessor()
return self._vlm_processor
diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.py
index 1b3bcc5a7..24e28cc53 100644
--- a/openviking/utils/resource_processor.py
+++ b/openviking/utils/resource_processor.py
@@ -7,17 +7,17 @@ Handles coordinated writes and self-iteration processes
as described in the OpenViking design document.
"""
-from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, Optional
from openviking.parse.tree_builder import TreeBuilder
-from openviking.utils.extractor import MediaStrategy
from openviking.storage import VikingDBManager
from openviking.storage.viking_fs import get_viking_fs
-from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter
from openviking.utils import get_logger
from openviking.utils.storage import StoragePath
+if TYPE_CHECKING:
+ from openviking.parse.vlm import VLMProcessor
+
logger = get_logger(__name__)
diff --git a/openviking/utils/skill_processor.py b/openviking/utils/skill_processor.py
index a705403c7..b84a79f34 100644
--- a/openviking/utils/skill_processor.py
+++ b/openviking/utils/skill_processor.py
@@ -6,13 +6,12 @@ Skill Processor for OpenViking.
Handles skill parsing, LLM generation, and storage operations.
"""
-from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional
from openviking.core.context import Context, ContextType, Vectorize
+from openviking.core.mcp_converter import is_mcp_format, mcp_to_skill
from openviking.core.skill_loader import SkillLoader
-from openviking.core.mcp_converter import mcp_to_skill, is_mcp_format
from openviking.storage import VikingDBManager
from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter
from openviking.storage.viking_fs import VikingFS
@@ -72,7 +71,7 @@ class SkillProcessor:
"allowed_tools": skill_dict.get("allowed_tools", []),
"tags": skill_dict.get("tags", []),
"source_path": skill_dict.get("source_path", ""),
- }
+ },
)
context.set_vectorize(Vectorize(text=context.abstract))
diff --git a/openviking/utils/uri.py b/openviking/utils/uri.py
index 678ae4d53..1c994148a 100644
--- a/openviking/utils/uri.py
+++ b/openviking/utils/uri.py
@@ -108,7 +108,7 @@ class VikingURI:
True if matches, False otherwise
"""
return self.uri.startswith(prefix)
-
+
@property
def parent(self) -> Optional["VikingURI"]:
"""
@@ -126,7 +126,7 @@ class VikingURI:
if scheme_end == -1:
return None
- after_scheme = uri[scheme_end + len(scheme_sep):]
+ after_scheme = uri[scheme_end + len(scheme_sep) :]
# If no / in after_scheme, only scope exists, no path
if "/" not in after_scheme:
diff --git a/pyproject.toml b/pyproject.toml
index 8e3f16497..9cf093a22 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -41,8 +41,6 @@ dependencies = [
"pytest>=7.0.0",
"pytest-asyncio>=0.21.0",
"pytest-cov>=4.0.0",
- "black>=23.0.0",
- "isort>=5.12.0",
"mypy>=1.0.0",
"ruff>=0.1.0",
"sphinx>=7.0.0",
@@ -78,15 +76,6 @@ exclude = ["tests*", "docs*", "examples*"]
[tool.setuptools.package-data]
openviking = ["prompts/templates/**/*.yaml"]
-[tool.black]
-line-length = 100
-target-version = ["py39", "py310", "py311", "py312"]
-include = '\.pyi?$'
-
-[tool.isort]
-profile = "black"
-line_length = 100
-
[tool.mypy]
python_version = "3.9"
warn_return_any = false
@@ -109,6 +98,8 @@ addopts = "-v --cov=openviking --cov-report=term-missing"
[tool.ruff]
line-length = 100
+exclude = ["third_party"]
+target-version = "py39"
[tool.ruff.lint]
select = [
@@ -132,3 +123,10 @@ ignore = [
[tool.ruff.lint.per-file-ignores]
"__init__.py" = ["F401"] # Allow unused imports in __init__.py
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+skip-magic-trailing-comma = false
+line-ending = "auto"
+
diff --git a/setup.py b/setup.py
index e1fba9d47..cc72f84ce 100644
--- a/setup.py
+++ b/setup.py
@@ -1,8 +1,11 @@
import os
-import sys
import shutil
+import sys
+import sysconfig
from pathlib import Path
-from setuptools import setup, find_packages, Extension
+
+import pybind11
+from setuptools import Extension, find_packages, setup
from setuptools.command.build_ext import build_ext
CMAKE_PATH = shutil.which("cmake") or "cmake"
@@ -34,7 +37,7 @@ class CMakeBuildExtension(build_ext):
# Paths
binary_name = "agfs-server.exe" if sys.platform == "win32" else "agfs-server"
agfs_server_dir = Path("third_party/agfs/agfs-server").resolve()
-
+
# Target in source tree (for development/install)
agfs_bin_dir = Path("openviking/bin").resolve()
agfs_target_binary = agfs_bin_dir / binary_name
@@ -43,34 +46,47 @@ class CMakeBuildExtension(build_ext):
if agfs_server_dir.exists() and shutil.which("go"):
print("Building AGFS server from source...")
import subprocess
+
try:
- build_args = ["go", "build", "-o", f"build/{binary_name}", "cmd/server/main.go"] if sys.platform == "win32" else ["make", "build"]
-
+ build_args = (
+ ["go", "build", "-o", f"build/{binary_name}", "cmd/server/main.go"]
+ if sys.platform == "win32"
+ else ["make", "build"]
+ )
+
subprocess.run(
build_args,
cwd=str(agfs_server_dir),
check=True,
stdout=subprocess.PIPE,
- stderr=subprocess.PIPE
+ stderr=subprocess.PIPE,
)
agfs_built_binary = agfs_server_dir / "build" / binary_name
if agfs_built_binary.exists():
self._copy_binary(agfs_built_binary, agfs_target_binary)
- print(f"[OK] AGFS server built successfully from source")
+ print("[OK] AGFS server built successfully from source")
else:
- raise FileNotFoundError(f"Build succeeded but binary not found at {agfs_built_binary}")
+ raise FileNotFoundError(
+ f"Build succeeded but binary not found at {agfs_built_binary}"
+ )
except (subprocess.CalledProcessError, Exception) as e:
error_msg = f"Failed to build AGFS from source: {e}"
if isinstance(e, subprocess.CalledProcessError):
- if e.stdout: error_msg += f"\nBuild stdout:\n{e.stdout.decode('utf-8', errors='replace')}"
- if e.stderr: error_msg += f"\nBuild stderr:\n{e.stderr.decode('utf-8', errors='replace')}"
+ if e.stdout:
+ error_msg += (
+ f"\nBuild stdout:\n{e.stdout.decode('utf-8', errors='replace')}"
+ )
+ if e.stderr:
+ error_msg += (
+ f"\nBuild stderr:\n{e.stderr.decode('utf-8', errors='replace')}"
+ )
raise RuntimeError(error_msg)
else:
if not agfs_server_dir.exists():
- raise FileNotFoundError(f"AGFS source directory not found at {agfs_server_dir}")
+ raise FileNotFoundError(f"AGFS source directory not found at {agfs_server_dir}")
else:
- raise RuntimeError("Go compiler not found. Please install Go to build AGFS server.")
+ raise RuntimeError("Go compiler not found. Please install Go to build AGFS server.")
# 2. Ensure AGFS binary is copied to the build directory (where wheel is packaged from)
if self.build_lib:
@@ -89,7 +105,7 @@ class CMakeBuildExtension(build_ext):
cmake_args = [
f"-S{Path(ENGINE_SOURCE_DIR).resolve()}",
f"-B{build_dir}",
- f"-DCMAKE_BUILD_TYPE=Release",
+ "-DCMAKE_BUILD_TYPE=Release",
f"-DPY_OUTPUT_DIR={ext_dir}",
"-DCMAKE_VERBOSE_MAKEFILE=ON",
"-DCMAKE_INSTALL_RPATH=$ORIGIN",
@@ -108,17 +124,10 @@ class CMakeBuildExtension(build_ext):
self.spawn([self.cmake_executable] + cmake_args)
- build_args = [
- "--build", str(build_dir),
- "--config", "Release",
- f"-j{os.cpu_count() or 4}"
- ]
+ build_args = ["--build", str(build_dir), "--config", "Release", f"-j{os.cpu_count() or 4}"]
self.spawn([self.cmake_executable] + build_args)
-import sysconfig
-import pybind11
-
setup(
ext_modules=[
Extension(
diff --git a/tests/client/test_file_operations.py b/tests/client/test_file_operations.py
index d5dd0b741..99415f205 100644
--- a/tests/client/test_file_operations.py
+++ b/tests/client/test_file_operations.py
@@ -24,15 +24,12 @@ class TestRm:
uris = await client.tree(result["root_uri"])
for data in uris:
- if not data['isDir']:
- await client.rm(data['uri'])
- with pytest.raises(Exception):
- await client.read(data['uri'])
-
-
- async def test_rm_directory_recursive(
- self, client: AsyncOpenViking, sample_directory: Path
- ):
+ if not data["isDir"]:
+ await client.rm(data["uri"])
+ with pytest.raises(Exception): # noqa: B017
+ await client.read(data["uri"])
+
+ async def test_rm_directory_recursive(self, client: AsyncOpenViking, sample_directory: Path):
"""Test recursive directory deletion"""
# Add files from directory first
for f in sample_directory.glob("**/*.txt"):
@@ -41,10 +38,10 @@ class TestRm:
# Get resource directory
entries = await client.ls("viking://resources/")
for data in entries:
- if data['isDir']:
- dir_uri = data['uri']
+ if data["isDir"]:
+ dir_uri = data["uri"]
await client.rm(dir_uri, recursive=True)
- with pytest.raises(Exception):
+ with pytest.raises(Exception): # noqa: B017
await client.stat(dir_uri)
@@ -62,7 +59,7 @@ class TestMv:
new_uri = "viking://resources/moved/"
await client.mv(uri, new_uri)
# Verify original location does not exist
- with pytest.raises(Exception):
+ with pytest.raises(Exception): # noqa: B017
await client.stat(uri)
await client.stat(new_uri)
@@ -85,11 +82,7 @@ class TestGrep:
"""Test case insensitive search"""
client, uri = client_with_resource
- result = await client.grep(
- uri,
- pattern="SAMPLE",
- case_insensitive=True
- )
+ result = await client.grep(uri, pattern="SAMPLE", case_insensitive=True)
print(result)
assert isinstance(result, dict)
assert "matches" in result and result["count"] > 0
@@ -120,10 +113,7 @@ class TestGlob:
client, uri = client_with_resource
parent_uri = "/".join(uri.split("/")[:-1]) + "/"
- result = await client.glob(
- pattern="*.md",
- uri=parent_uri
- )
+ result = await client.glob(pattern="*.md", uri=parent_uri)
assert isinstance(result, dict)
assert "matches" in result and result["count"] > 0
diff --git a/tests/client/test_filesystem.py b/tests/client/test_filesystem.py
index e0830311c..ca3572238 100644
--- a/tests/client/test_filesystem.py
+++ b/tests/client/test_filesystem.py
@@ -56,15 +56,15 @@ class TestRead:
entries = await client.tree(uri)
content = ""
for e in entries:
- if not e['isDir']:
- content = await client.read(e['uri'])
+ if not e["isDir"]:
+ content = await client.read(e["uri"])
assert isinstance(content, str)
assert len(content) > 0
- assert "Sample Document" in content
+ assert "Sample Document" in content
async def test_read_nonexistent_file(self, client: AsyncOpenViking):
"""Test reading nonexistent file"""
- with pytest.raises(Exception):
+ with pytest.raises(Exception): # noqa: B017
await client.read("viking://nonexistent/file.txt")
diff --git a/tests/client/test_import_export.py b/tests/client/test_import_export.py
index 001588917..e4b1b9f13 100644
--- a/tests/client/test_import_export.py
+++ b/tests/client/test_import_export.py
@@ -5,8 +5,6 @@
from pathlib import Path
-import pytest
-
from openviking import AsyncOpenViking
@@ -33,10 +31,7 @@ class TestExportOvpack:
# Export entire resource directory
export_path = temp_dir / "dir_export.ovpack"
- result = await client.export_ovpack(
- "viking://resources/",
- str(export_path)
- )
+ result = await client.export_ovpack("viking://resources/", str(export_path))
assert isinstance(result, str)
@@ -54,9 +49,7 @@ class TestImportOvpack:
# Import to new location
import_uri = await client.import_ovpack(
- str(export_path),
- "viking://resources/imported/",
- vectorize=False
+ str(export_path), "viking://resources/imported/", vectorize=False
)
assert isinstance(import_uri, str)
@@ -72,39 +65,30 @@ class TestImportOvpack:
# First import
await client.import_ovpack(
- str(export_path),
- "viking://resources/force_test/",
- vectorize=False
+ str(export_path), "viking://resources/force_test/", vectorize=False
)
# Second force import (overwrite)
import_uri = await client.import_ovpack(
- str(export_path),
- "viking://resources/force_test/",
- force=True,
- vectorize=False
+ str(export_path), "viking://resources/force_test/", force=True, vectorize=False
)
assert isinstance(import_uri, str)
-
async def test_import_export_roundtrip(
self, client: AsyncOpenViking, sample_markdown_file: Path, temp_dir: Path
):
"""Test export-import roundtrip"""
# Add resource
- result = await client.add_resource(
- path=str(sample_markdown_file),
- reason="Roundtrip test"
- )
+ result = await client.add_resource(path=str(sample_markdown_file), reason="Roundtrip test")
original_uri = result["root_uri"]
# Read original content
original_content = ""
entries = await client.tree(original_uri)
for e in entries:
- if not e['isDir']:
- original_content = await client.read(e['uri'])
+ if not e["isDir"]:
+ original_content = await client.read(e["uri"])
# Export
export_path = temp_dir / "roundtrip.ovpack"
@@ -115,17 +99,15 @@ class TestImportOvpack:
# Import
import_uri = await client.import_ovpack(
- str(export_path),
- "viking://resources/roundtrip/",
- vectorize=False
+ str(export_path), "viking://resources/roundtrip/", vectorize=False
)
# Read imported content
imported_content = ""
entries = await client.tree(import_uri)
for e in entries:
- if not e['isDir']:
- imported_content = await client.read(e['uri'])
+ if not e["isDir"]:
+ imported_content = await client.read(e["uri"])
# Verify content consistency
assert original_content == imported_content
diff --git a/tests/client/test_lifecycle.py b/tests/client/test_lifecycle.py
index 2e1f8d7ee..13a397802 100644
--- a/tests/client/test_lifecycle.py
+++ b/tests/client/test_lifecycle.py
@@ -5,9 +5,6 @@
from pathlib import Path
-import pytest
-import pytest_asyncio
-
from openviking import AsyncOpenViking
diff --git a/tests/client/test_relations.py b/tests/client/test_relations.py
index 9d73e7972..209838c5f 100644
--- a/tests/client/test_relations.py
+++ b/tests/client/test_relations.py
@@ -3,10 +3,6 @@
"""Relation tests"""
-import pytest
-
-from openviking import AsyncOpenViking
-
class TestLink:
"""Test link creating relations"""
@@ -16,11 +12,7 @@ class TestLink:
client, uri = client_with_resource
target_uri = "viking://resources/target/"
- await client.link(
- from_uri=uri,
- uris=target_uri,
- reason="Test link"
- )
+ await client.link(from_uri=uri, uris=target_uri, reason="Test link")
relations = await client.relations(uri)
assert any(r.get("uri") == target_uri for r in relations)
@@ -28,16 +20,9 @@ class TestLink:
async def test_link_multiple_uris(self, client_with_resource):
"""Test creating multiple relations"""
client, uri = client_with_resource
- target_uris = [
- "viking://resources/target1/",
- "viking://resources/target2/"
- ]
+ target_uris = ["viking://resources/target1/", "viking://resources/target2/"]
- await client.link(
- from_uri=uri,
- uris=target_uris,
- reason="Test multiple links"
- )
+ await client.link(from_uri=uri, uris=target_uris, reason="Test multiple links")
relations = await client.relations(uri)
for target in target_uris:
@@ -49,11 +34,7 @@ class TestLink:
target_uri = "viking://resources/reason_test/"
reason = "This is a test reason for the link"
- await client.link(
- from_uri=uri,
- uris=target_uri,
- reason=reason
- )
+ await client.link(from_uri=uri, uris=target_uri, reason=reason)
relations = await client.relations(uri)
link = next((r for r in relations if r.get("uri") == target_uri), None)
@@ -88,10 +69,7 @@ class TestUnlink:
client, uri = client_with_resource
# Should not raise exception
- await client.unlink(
- from_uri=uri,
- uri="viking://nonexistent/"
- )
+ await client.unlink(from_uri=uri, uri="viking://nonexistent/")
class TestRelations:
diff --git a/tests/client/test_resource_management.py b/tests/client/test_resource_management.py
index 269f23a61..c82fd8a42 100644
--- a/tests/client/test_resource_management.py
+++ b/tests/client/test_resource_management.py
@@ -5,22 +5,15 @@
from pathlib import Path
-import pytest
-
from openviking import AsyncOpenViking
class TestAddResource:
"""Test add_resource"""
- async def test_add_resource_success(
- self, client: AsyncOpenViking, sample_markdown_file: Path
- ):
+ async def test_add_resource_success(self, client: AsyncOpenViking, sample_markdown_file: Path):
"""Test successful resource addition"""
- result = await client.add_resource(
- path=str(sample_markdown_file),
- reason="Test resource"
- )
+ result = await client.add_resource(path=str(sample_markdown_file), reason="Test resource")
assert "root_uri" in result
assert result["root_uri"].startswith("viking://")
@@ -44,9 +37,7 @@ class TestAddResource:
):
"""Test adding resource without waiting (async mode)"""
result = await client.add_resource(
- path=str(sample_markdown_file),
- reason="Test resource",
- wait=False
+ path=str(sample_markdown_file), reason="Test resource", wait=False
)
assert "root_uri" in result
@@ -61,7 +52,7 @@ class TestAddResource:
result = await client.add_resource(
path=str(sample_markdown_file),
target="viking://resources/custom/",
- reason="Test resource"
+ reason="Test resource",
)
assert "root_uri" in result
@@ -69,12 +60,9 @@ class TestAddResource:
async def test_add_resource_file_not_found(self, client: AsyncOpenViking):
"""Test adding nonexistent file"""
-
- res = await client.add_resource(
- path="/nonexistent/file.txt",
- reason="Test"
- )
-
+
+ res = await client.add_resource(path="/nonexistent/file.txt", reason="Test")
+
assert "errors" in res and len(res["errors"]) > 0
@@ -85,10 +73,7 @@ class TestWaitProcessed:
self, client: AsyncOpenViking, sample_markdown_file: Path
):
"""Test waiting for processing to complete"""
- await client.add_resource(
- path=str(sample_markdown_file),
- reason="Test"
- )
+ await client.add_resource(path=str(sample_markdown_file), reason="Test")
status = await client.wait_processed(timeout=60.0)
diff --git a/tests/client/test_search.py b/tests/client/test_search.py
index fa97139dc..ff661a25a 100644
--- a/tests/client/test_search.py
+++ b/tests/client/test_search.py
@@ -3,9 +3,6 @@
"""Search tests"""
-import pytest
-
-from openviking import AsyncOpenViking
from openviking.message import TextPart
@@ -23,34 +20,25 @@ class TestFind:
assert hasattr(result, "skills")
assert hasattr(result, "total")
-
"""Test limiting result count"""
result = await client.find(query="test", limit=5)
assert len(result.resources) <= 5
"""Test search with target URI"""
- result = await client.find(
- query="sample",
- target_uri=uri
- )
+ result = await client.find(query="sample", target_uri=uri)
assert hasattr(result, "resources")
"""Test score threshold filtering"""
- result = await client.find(
- query="sample document",
- score_threshold=0.1
- )
+ result = await client.find(query="sample document", score_threshold=0.1)
# Verify all results have score >= threshold
for res in result.resources:
assert res.score >= 0.1
"""Test no matching results"""
- result = await client.find(
- query="completely_random_nonexistent_query_xyz123"
- )
+ result = await client.find(query="completely_random_nonexistent_query_xyz123")
assert result.total >= 0
@@ -71,27 +59,18 @@ class TestSearch:
# Add some messages to establish context
session.add_message("user", [TextPart("I need help with testing")])
- result = await client.search(
- query="testing help",
- session=session
- )
+ result = await client.search(query="testing help", session=session)
assert hasattr(result, "resources")
"""Test limiting result count"""
- result = await client.search(
- query="sample",
- limit=3
- )
+ result = await client.search(query="sample", limit=3)
assert len(result.resources) <= 3
"""Test complex search with target URI"""
parent_uri = "/".join(uri.split("/")[:-1]) + "/"
- result = await client.search(
- query="sample",
- target_uri=parent_uri
- )
+ result = await client.search(query="sample", target_uri=parent_uri)
assert hasattr(result, "resources")
diff --git a/tests/client/test_skill_management.py b/tests/client/test_skill_management.py
index e98f15934..40ce64718 100644
--- a/tests/client/test_skill_management.py
+++ b/tests/client/test_skill_management.py
@@ -4,9 +4,6 @@
"""Skill management tests"""
from pathlib import Path
-import tempfile
-
-import pytest
from openviking import AsyncOpenViking
@@ -18,7 +15,8 @@ class TestAddSkill:
"""Test adding skill from file"""
# Create skill file in SKILL.md format
skill_file = temp_dir / "test_skill.md"
- skill_file.write_text("""---
+ skill_file.write_text(
+ """---
name: test-skill
description: A test skill for unit testing
tags:
@@ -38,7 +36,8 @@ Use this skill when you need to test skill functionality.
1. Step one: Initialize the skill
2. Step two: Execute the skill
3. Step three: Verify the result
-""")
+"""
+ )
result = await client.add_skill(data=skill_file)
@@ -71,14 +70,9 @@ This skill was created from a string.
"description": "A test MCP tool",
"inputSchema": {
"type": "object",
- "properties": {
- "query": {
- "type": "string",
- "description": "The search query"
- }
- },
- "required": ["query"]
- }
+ "properties": {"query": {"type": "string", "description": "The search query"}},
+ "required": ["query"],
+ },
}
result = await client.add_skill(data=mcp_tool)
@@ -91,7 +85,8 @@ This skill was created from a string.
skill_dir.mkdir()
# Create SKILL.md
- (skill_dir / "SKILL.md").write_text("""---
+ (skill_dir / "SKILL.md").write_text(
+ """---
name: dir-skill
description: A skill from directory
tags:
@@ -102,7 +97,8 @@ tags:
## Instructions
This skill was loaded from a directory.
-""")
+"""
+ )
# Create auxiliary file
(skill_dir / "reference.md").write_text("# Reference\nAdditional reference content.")
@@ -120,7 +116,8 @@ class TestSkillSearch:
"""Test searching skills"""
# Add skill first
skill_file = temp_dir / "search_skill.md"
- skill_file.write_text("""---
+ skill_file.write_text(
+ """---
name: search-test-skill
description: A skill for testing search functionality
tags:
@@ -132,7 +129,8 @@ tags:
## Instructions
Use this skill to test search functionality.
-""")
+"""
+ )
await client.add_skill(data=skill_file)
# Search skills
diff --git a/tests/conftest.py b/tests/conftest.py
index 668bb57fc..cff8681cf 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -13,7 +13,6 @@ import pytest_asyncio
from openviking import AsyncOpenViking
-
# Test data root directory
TEST_ROOT = Path(__file__).parent
TEST_TMP_DIR = TEST_ROOT / ".tmp"
@@ -56,7 +55,8 @@ def sample_text_file(temp_dir: Path) -> Path:
def sample_markdown_file(temp_dir: Path) -> Path:
"""Create sample Markdown file"""
file_path = temp_dir / "sample.md"
- file_path.write_text("""# Sample Document
+ file_path.write_text(
+ """# Sample Document
## Introduction
This is a sample markdown document for testing OpenViking.
@@ -68,7 +68,8 @@ This is a sample markdown document for testing OpenViking.
## Usage
Use this document to test various OpenViking functionalities.
-""")
+"""
+ )
return file_path
@@ -76,7 +77,8 @@ Use this document to test various OpenViking functionalities.
def sample_skill_file(temp_dir: Path) -> Path:
"""Create sample skill file in SKILL.md format"""
file_path = temp_dir / "sample_skill.md"
- file_path.write_text("""---
+ file_path.write_text(
+ """---
name: sample-skill
description: A sample skill for testing OpenViking skill management
tags:
@@ -96,7 +98,8 @@ Use this skill when you need to test skill functionality.
1. Step one: Initialize the skill
2. Step two: Execute the skill
3. Step three: Verify the result
-""")
+"""
+ )
return file_path
@@ -122,7 +125,8 @@ def sample_files(temp_dir: Path) -> list[Path]:
files = []
for i in range(3):
file_path = temp_dir / f"batch_file_{i}.md"
- file_path.write_text(f"""# Batch File {i}
+ file_path.write_text(
+ f"""# Batch File {i}
## Content
This is batch file number {i} for testing batch operations.
@@ -131,22 +135,21 @@ This is batch file number {i} for testing batch operations.
- batch
- test
- file{i}
-""")
+"""
+ )
files.append(file_path)
return files
# ============ Client Fixtures ============
+
@pytest_asyncio.fixture(scope="function")
async def client(test_data_dir: Path) -> AsyncGenerator[AsyncOpenViking, None]:
"""Create initialized OpenViking client"""
await AsyncOpenViking.reset()
- client = AsyncOpenViking(
- path=str(test_data_dir),
- user="test_user"
- )
+ client = AsyncOpenViking(path=str(test_data_dir), user="test_user")
await client.initialize()
yield client
@@ -160,10 +163,7 @@ async def uninitialized_client(test_data_dir: Path) -> AsyncGenerator[AsyncOpenV
"""Create uninitialized OpenViking client (for testing initialization flow)"""
await AsyncOpenViking.reset()
- client = AsyncOpenViking(
- path=str(test_data_dir),
- user="test_user"
- )
+ client = AsyncOpenViking(path=str(test_data_dir), user="test_user")
yield client
@@ -176,14 +176,11 @@ async def uninitialized_client(test_data_dir: Path) -> AsyncGenerator[AsyncOpenV
@pytest_asyncio.fixture(scope="function")
async def client_with_resource_sync(
- client: AsyncOpenViking,
- sample_markdown_file: Path
+ client: AsyncOpenViking, sample_markdown_file: Path
) -> AsyncGenerator[tuple[AsyncOpenViking, str], None]:
"""Create client with resource (sync mode, wait for vectorization)"""
result = await client.add_resource(
- path=str(sample_markdown_file),
- reason="Test resource",
- wait=True
+ path=str(sample_markdown_file), reason="Test resource", wait=True
)
uri = result.get("root_uri", "")
@@ -192,13 +189,9 @@ async def client_with_resource_sync(
@pytest_asyncio.fixture(scope="function")
async def client_with_resource(
- client: AsyncOpenViking,
- sample_markdown_file: Path
+ client: AsyncOpenViking, sample_markdown_file: Path
) -> AsyncGenerator[tuple[AsyncOpenViking, str], None]:
"""Create client with resource (async mode, no wait for vectorization)"""
- result = await client.add_resource(
- path=str(sample_markdown_file),
- reason="Test resource"
- )
+ result = await client.add_resource(path=str(sample_markdown_file), reason="Test resource")
uri = result.get("root_uri", "")
yield client, uri
diff --git a/tests/integration/test_full_workflow.py b/tests/integration/test_full_workflow.py
index 269e4f937..3e2c1107d 100644
--- a/tests/integration/test_full_workflow.py
+++ b/tests/integration/test_full_workflow.py
@@ -5,7 +5,6 @@
from pathlib import Path
-import pytest
import pytest_asyncio
from openviking import AsyncOpenViking
@@ -17,10 +16,7 @@ async def integration_client(test_data_dir: Path):
"""Integration test client"""
await AsyncOpenViking.reset()
- client = AsyncOpenViking(
- path=str(test_data_dir),
- user="integration_test_user"
- )
+ client = AsyncOpenViking(path=str(test_data_dir), user="integration_test_user")
await client.initialize()
yield client
@@ -59,12 +55,7 @@ class TestResourceToSearchWorkflow:
client = integration_client
# 1. Add resource
- add_result = await client.add_resource(
- path=str(sample_markdown_file),
- reason="Workflow test",
- wait=True
- )
- uri = add_result["root_uri"]
+ await client.add_resource(path=str(sample_markdown_file), reason="Workflow test", wait=True)
# 2. Search
search_result = await client.find(query="sample document")
@@ -77,10 +68,10 @@ class TestResourceToSearchWorkflow:
else:
res = await client.tree(search_result.resources[0].uri)
for data in res:
- if not data['isDir']:
- content = await client.read(data['uri'])
+ if not data["isDir"]:
+ content = await client.read(data["uri"])
assert len(content) > 0
-
+
class TestSessionWorkflow:
"""Session management full workflow"""
@@ -92,12 +83,9 @@ class TestSessionWorkflow:
client = integration_client
# 1. Add resource
- result = await client.add_resource(
- path=str(sample_markdown_file),
- reason="Session workflow test",
- wait=True
+ await client.add_resource(
+ path=str(sample_markdown_file), reason="Session workflow test", wait=True
)
- resource_uri = result["root_uri"]
# 2. Create session
session = client.session(session_id="workflow_test_session")
@@ -166,8 +154,8 @@ class TestImportExportWorkflow:
original_content = ""
entries = await client.tree(original_uri)
for data in entries:
- if not data['isDir']:
- original_content += await client.read(data['uri'])
+ if not data["isDir"]:
+ original_content += await client.read(data["uri"])
# 3. Export
export_path = temp_dir / "workflow_export.ovpack"
@@ -179,17 +167,15 @@ class TestImportExportWorkflow:
# 5. Import
import_uri = await client.import_ovpack(
- str(export_path),
- "viking://resources/imported/",
- vectorize=False
+ str(export_path), "viking://resources/imported/", vectorize=False
)
# 6. Verify content consistency
imported_content = ""
entries = await client.tree(import_uri)
for data in entries:
- if not data['isDir']:
- imported_content += await client.read(data['uri'])
+ if not data["isDir"]:
+ imported_content += await client.read(data["uri"])
assert original_content == imported_content
@@ -242,8 +228,7 @@ class TestFullEndToEndWorkflow:
# Import to new location
import_uri = await client.import_ovpack(
- str(export_path),
- "viking://resources/e2e_imported/"
+ str(export_path), "viking://resources/e2e_imported/"
)
# Verify import success
diff --git a/tests/misc/test_config_validation.py b/tests/misc/test_config_validation.py
index 147ae6064..35e50b0b1 100644
--- a/tests/misc/test_config_validation.py
+++ b/tests/misc/test_config_validation.py
@@ -4,11 +4,13 @@
"""Test if config validators work correctly"""
import sys
+
from openviking.utils.config.agfs_config import AGFSConfig
+from openviking.utils.config.embedding_config import EmbeddingConfig, EmbeddingModelConfig
from openviking.utils.config.vectordb_config import VectorDBBackendConfig
-from openviking.utils.config.embedding_config import EmbeddingModelConfig, EmbeddingConfig
from openviking.utils.config.vlm_config import VLMConfig
+
def test_agfs_validation():
"""Test AGFS config validation"""
print("=" * 60)
@@ -27,7 +29,7 @@ def test_agfs_validation():
print("\n2. Test invalid backend...")
try:
config = AGFSConfig(backend="invalid")
- print(f" Should fail but passed")
+ print(" Should fail but passed")
except ValueError as e:
print(f" Correctly raised exception: {e}")
@@ -35,7 +37,7 @@ def test_agfs_validation():
print("\n3. Test S3 backend missing required fields...")
try:
config = AGFSConfig(backend="s3")
- print(f" Should fail but passed")
+ print(" Should fail but passed")
except ValueError as e:
print(f" Correctly raised exception: {e}")
@@ -47,12 +49,13 @@ def test_agfs_validation():
s3_bucket="my-bucket",
s3_region="us-west-1",
s3_access_key="fake-access-key-for-testing",
- s3_secret_key="fake-secret-key-for-testing-12345"
+ s3_secret_key="fake-secret-key-for-testing-12345",
)
- print(f" Pass")
+ print(" Pass")
except ValueError as e:
print(f" Fail: {e}")
+
def test_vectordb_validation():
"""Test VectorDB config validation"""
print("\n" + "=" * 60)
@@ -62,34 +65,31 @@ def test_vectordb_validation():
# Test 1: local backend missing path
print("\n1. Test local backend missing path...")
try:
- config = VectorDBBackendConfig(backend="local", path=None)
- print(f" Should fail but passed")
+ _ = VectorDBBackendConfig(backend="local", path=None)
+ print(" Should fail but passed")
except ValueError as e:
print(f" Correctly raised exception: {e}")
# Test 2: http backend missing url
print("\n2. Test http backend missing url...")
try:
- config = VectorDBBackendConfig(backend="http", url=None)
- print(f" Should fail but passed")
+ _ = VectorDBBackendConfig(backend="http", url=None)
+ print(" Should fail but passed")
except ValueError as e:
print(f" Correctly raised exception: {e}")
# Test 3: volcengine backend complete config
print("\n3. Test volcengine backend complete config...")
try:
- config = VectorDBBackendConfig(
+ _ = VectorDBBackendConfig(
backend="volcengine",
- volcengine={
- "ak": "test_ak",
- "sk": "test_sk",
- "region": "cn-beijing"
- }
+ volcengine={"ak": "test_ak", "sk": "test_sk", "region": "cn-beijing"},
)
- print(f" Pass")
+ print(" Pass")
except ValueError as e:
print(f" Fail: {e}")
+
def test_embedding_validation():
"""Test Embedding config validation"""
print("\n" + "=" * 60)
@@ -99,36 +99,33 @@ def test_embedding_validation():
# Test 1: no embedder config
print("\n1. Test no embedder config...")
try:
- config = EmbeddingConfig()
- print(f" Should fail but passed")
+ _ = EmbeddingConfig()
+ print(" Should fail but passed")
except ValueError as e:
print(f" Correctly raised exception: {e}")
# Test 2: OpenAI provider missing api_key
print("\n2. Test OpenAI provider missing api_key...")
try:
- config = EmbeddingConfig(
- dense=EmbeddingModelConfig(
- provider="openai",
- model="text-embedding-3-small"
- )
+ _ = EmbeddingConfig(
+ dense=EmbeddingModelConfig(provider="openai", model="text-embedding-3-small")
)
- print(f" Should fail but passed")
+ print(" Should fail but passed")
except ValueError as e:
print(f" Correctly raised exception: {e}")
# Test 3: OpenAI provider complete config
print("\n3. Test OpenAI provider complete config...")
try:
- config = EmbeddingConfig(
+ _ = EmbeddingConfig(
dense=EmbeddingModelConfig(
provider="openai",
model="text-embedding-3-small",
api_key="fake-api-key-for-testing",
- dimension=1536
+ dimension=1536,
)
)
- print(f" Pass")
+ print(" Pass")
except ValueError as e:
print(f" Fail: {e}")
@@ -136,29 +133,27 @@ def test_embedding_validation():
print("\n4. Test Embedding Provider/Backend sync...")
# Case A: Only backend provided -> provider should be synced
config_a = EmbeddingModelConfig(
- backend="openai",
- model="text-embedding-3-small",
- api_key="test-key",
- dimension=1536
+ backend="openai", model="text-embedding-3-small", api_key="test-key", dimension=1536
)
if config_a.provider == "openai":
- print(f" Pass (backend='openai' -> provider='openai')")
+ print(" Pass (backend='openai' -> provider='openai')")
else:
print(f" Fail (backend='openai' -> provider='{config_a.provider}')")
# Case B: Both provided -> provider takes precedence
config_b = EmbeddingModelConfig(
provider="volcengine",
- backend="openai", # Conflicting backend
+ backend="openai", # Conflicting backend
model="doubao",
api_key="test-key",
- dimension=1024
+ dimension=1024,
)
if config_b.provider == "volcengine":
- print(f" Pass (provider='volcengine' priority over backend='openai')")
+ print(" Pass (provider='volcengine' priority over backend='openai')")
else:
print(f" Fail (provider='volcengine' should have priority, got '{config_b.provider}')")
+
def test_vlm_validation():
"""Test VLM config validation"""
print("\n" + "=" * 60)
@@ -168,56 +163,46 @@ def test_vlm_validation():
# Test 1: VLM not configured (optional)
print("\n1. Test VLM not configured (optional)...")
try:
- config = VLMConfig()
- print(f" Pass (VLM is optional)")
+ _ = VLMConfig()
+ print(" Pass (VLM is optional)")
except ValueError as e:
print(f" Fail: {e}")
# Test 2: VLM partial config (has model but no api_key)
print("\n2. Test VLM partial config...")
try:
- config = VLMConfig(model="gpt-4")
- print(f" Should fail but passed")
+ _ = VLMConfig(model="gpt-4")
+ print(" Should fail but passed")
except ValueError as e:
print(f" Correctly raised exception: {e}")
# Test 3: VLM complete config
print("\n3. Test VLM complete config...")
try:
- config = VLMConfig(
- model="gpt-4",
- api_key="fake-api-key-for-testing",
- provider="openai"
- )
- print(f" Pass")
+ _ = VLMConfig(model="gpt-4", api_key="fake-api-key-for-testing", provider="openai")
+ print(" Pass")
except ValueError as e:
print(f" Fail: {e}")
# Test 4: VLM Provider/Backend sync
print("\n4. Test VLM Provider/Backend sync...")
# Case A: Only backend provided -> provider should be synced
- config_a = VLMConfig(
- backend="openai",
- model="gpt-4",
- api_key="test-key"
- )
+ config_a = VLMConfig(backend="openai", model="gpt-4", api_key="test-key")
if config_a.provider == "openai":
- print(f" Pass (backend='openai' -> provider='openai')")
+ print(" Pass (backend='openai' -> provider='openai')")
else:
print(f" Fail (backend='openai' -> provider='{config_a.provider}')")
# Case B: Both provided -> provider takes precedence
config_b = VLMConfig(
- provider="volcengine",
- backend="openai",
- model="doubao",
- api_key="test-key"
+ provider="volcengine", backend="openai", model="doubao", api_key="test-key"
)
if config_b.provider == "volcengine":
- print(f" Pass (provider='volcengine' priority over backend='openai')")
+ print(" Pass (provider='volcengine' priority over backend='openai')")
else:
print(f" Fail (provider='volcengine' should have priority, got '{config_b.provider}')")
+
if __name__ == "__main__":
print("\nStarting config validator tests...\n")
@@ -234,5 +219,6 @@ if __name__ == "__main__":
except Exception as e:
print(f"\nUnexpected error during tests: {e}")
import traceback
+
traceback.print_exc()
sys.exit(1)
diff --git a/tests/misc/test_vikingdb_observer.py b/tests/misc/test_vikingdb_observer.py
index 156b88024..0cfbfa168 100644
--- a/tests/misc/test_vikingdb_observer.py
+++ b/tests/misc/test_vikingdb_observer.py
@@ -6,6 +6,7 @@ Test VikingDBObserver functionality
"""
import asyncio
+
import openviking as ov
diff --git a/tests/session/conftest.py b/tests/session/conftest.py
index 0b1b61f7a..1395738f2 100644
--- a/tests/session/conftest.py
+++ b/tests/session/conftest.py
@@ -5,12 +5,11 @@
from typing import AsyncGenerator
-import pytest
import pytest_asyncio
from openviking import AsyncOpenViking
-from openviking.session import Session
from openviking.message import TextPart, ToolPart
+from openviking.session import Session
@pytest_asyncio.fixture(scope="function")
@@ -42,7 +41,7 @@ async def session_with_messages(client: AsyncOpenViking) -> AsyncGenerator[Sessi
@pytest_asyncio.fixture(scope="function")
async def session_with_tool_call(
- client: AsyncOpenViking
+ client: AsyncOpenViking,
) -> AsyncGenerator[tuple[Session, str, str], None]:
"""Create Session with tool call"""
session = client.session(session_id="test_session_with_tool")
@@ -54,12 +53,9 @@ async def session_with_tool_call(
tool_uri=f"viking://session/{session.session_id}/tools/{tool_id}",
skill_uri="viking://agent/skills/test_skill",
tool_input={"param": "value"},
- tool_status="running"
+ tool_status="running",
)
- msg = session.add_message("assistant", [
- TextPart("Executing tool..."),
- tool_part
- ])
+ msg = session.add_message("assistant", [TextPart("Executing tool..."), tool_part])
yield session, msg.id, tool_id
diff --git a/tests/session/test_session_commit.py b/tests/session/test_session_commit.py
index 1dcf230bb..0f6a90d9a 100644
--- a/tests/session/test_session_commit.py
+++ b/tests/session/test_session_commit.py
@@ -3,11 +3,9 @@
"""Commit tests"""
-import pytest
-
from openviking import AsyncOpenViking
-from openviking.session import Session
from openviking.message import TextPart
+from openviking.session import Session
class TestCommit:
@@ -21,7 +19,9 @@ class TestCommit:
assert result.get("status") == "committed"
assert "session_id" in result
- async def test_commit_extracts_memories(self, session_with_messages: Session, client: AsyncOpenViking):
+ async def test_commit_extracts_memories(
+ self, session_with_messages: Session, client: AsyncOpenViking
+ ):
"""Test commit extracts memories"""
result = session_with_messages.commit()
diff --git a/tests/session/test_session_context.py b/tests/session/test_session_context.py
index ecac52182..ceccf3698 100644
--- a/tests/session/test_session_context.py
+++ b/tests/session/test_session_context.py
@@ -3,11 +3,9 @@
"""Context retrieval tests"""
-import pytest
-
from openviking import AsyncOpenViking
-from openviking.session import Session
from openviking.message import TextPart
+from openviking.session import Session
class TestGetContextForSearch:
@@ -15,19 +13,14 @@ class TestGetContextForSearch:
async def test_get_context_basic(self, session_with_messages: Session):
"""Test basic context retrieval"""
- context = session_with_messages.get_context_for_search(
- query="testing help"
- )
+ context = session_with_messages.get_context_for_search(query="testing help")
assert isinstance(context, dict)
assert "summaries" in context or "recent_messages" in context
async def test_get_context_with_max_messages(self, session_with_messages: Session):
"""Test limiting max messages"""
- context = session_with_messages.get_context_for_search(
- query="test",
- max_messages=2
- )
+ context = session_with_messages.get_context_for_search(query="test", max_messages=2)
assert isinstance(context, dict)
if "recent_messages" in context:
@@ -45,10 +38,7 @@ class TestGetContextForSearch:
# Add more messages
session.add_message("user", [TextPart("Second message")])
- context = session.get_context_for_search(
- query="test",
- max_archives=1
- )
+ context = session.get_context_for_search(query="test", max_archives=1)
assert isinstance(context, dict)
diff --git a/tests/session/test_session_lifecycle.py b/tests/session/test_session_lifecycle.py
index 4b749dd89..a48c5d127 100644
--- a/tests/session/test_session_lifecycle.py
+++ b/tests/session/test_session_lifecycle.py
@@ -3,8 +3,6 @@
"""Session lifecycle tests"""
-import pytest
-
from openviking import AsyncOpenViking
from openviking.session import Session
@@ -46,7 +44,9 @@ class TestSessionCreate:
class TestSessionLoad:
"""Test Session loading"""
- async def test_load_existing_session(self, session_with_messages: Session, client: AsyncOpenViking):
+ async def test_load_existing_session(
+ self, session_with_messages: Session, client: AsyncOpenViking
+ ):
"""Test loading existing session"""
session_id = session_with_messages.session_id
diff --git a/tests/session/test_session_messages.py b/tests/session/test_session_messages.py
index 8fc9587cc..25f2267c7 100644
--- a/tests/session/test_session_messages.py
+++ b/tests/session/test_session_messages.py
@@ -3,10 +3,8 @@
"""Message management tests"""
-import pytest
-
+from openviking.message import ContextPart, TextPart, ToolPart
from openviking.session import Session
-from openviking.message import TextPart, ContextPart, ToolPart
class TestAddMessage:
@@ -31,10 +29,7 @@ class TestAddMessage:
async def test_add_message_with_multiple_parts(self, session: Session):
"""Test adding message with multiple parts"""
- parts = [
- TextPart("Here is some context:"),
- TextPart("And here is more text.")
- ]
+ parts = [TextPart("Here is some context:"), TextPart("And here is more text.")]
msg = session.add_message("assistant", parts)
assert len(msg.parts) == 2
@@ -46,8 +41,8 @@ class TestAddMessage:
ContextPart(
uri="viking://user/test/resources/doc.md",
context_type="resource",
- abstract="Some context abstract"
- )
+ abstract="Some context abstract",
+ ),
]
msg = session.add_message("assistant", parts)
@@ -61,12 +56,9 @@ class TestAddMessage:
tool_uri="viking://session/test/tools/tool_123",
skill_uri="viking://agent/skills/search",
tool_input={"query": "test"},
- tool_status="running"
+ tool_status="running",
)
- msg = session.add_message("assistant", [
- TextPart("Executing search..."),
- tool_part
- ])
+ msg = session.add_message("assistant", [TextPart("Executing search..."), tool_part])
assert len(msg.parts) == 2
@@ -91,7 +83,7 @@ class TestUpdateToolPart:
message_id=message_id,
tool_id=tool_id,
output="Tool execution completed successfully",
- status="completed"
+ status="completed",
)
# Verify tool status updated
@@ -107,7 +99,7 @@ class TestUpdateToolPart:
message_id=message_id,
tool_id=tool_id,
output="Tool execution failed: error message",
- status="failed"
+ status="failed",
)
# Verify tool status updated
diff --git a/tests/session/test_session_usage.py b/tests/session/test_session_usage.py
index 480043c75..d9cc5a11c 100644
--- a/tests/session/test_session_usage.py
+++ b/tests/session/test_session_usage.py
@@ -3,10 +3,8 @@
"""Usage record tests"""
-import pytest
-
-from openviking.session import Session
from openviking.message import TextPart
+from openviking.session import Session
class TestUsed:
@@ -18,10 +16,12 @@ class TestUsed:
session.add_message("user", [TextPart("Test message")])
# Record used contexts
- session.used(contexts=[
- "viking://user/test/resources/doc1.md",
- "viking://user/test/resources/doc2.md"
- ])
+ session.used(
+ contexts=[
+ "viking://user/test/resources/doc1.md",
+ "viking://user/test/resources/doc2.md",
+ ]
+ )
# Verify usage records
assert len(session.usage_records) > 0
@@ -30,10 +30,7 @@ class TestUsed:
"""Test recording used skill"""
session.add_message("user", [TextPart("Test message")])
- session.used(skill={
- "uri": "viking://agent/skills/search",
- "name": "search_skill"
- })
+ session.used(skill={"uri": "viking://agent/skills/search", "name": "search_skill"})
assert len(session.usage_records) > 0
@@ -43,10 +40,7 @@ class TestUsed:
session.used(
contexts=["viking://user/test/resources/doc.md"],
- skill={
- "uri": "viking://agent/skills/analyze",
- "name": "analyze_skill"
- }
+ skill={"uri": "viking://agent/skills/analyze", "name": "analyze_skill"},
)
assert len(session.usage_records) > 0
diff --git a/tests/vectordb/benchmark_stress.py b/tests/vectordb/benchmark_stress.py
index 1168a557c..016618c11 100644
--- a/tests/vectordb/benchmark_stress.py
+++ b/tests/vectordb/benchmark_stress.py
@@ -1,16 +1,14 @@
import argparse
+import os
import random
-import time
-import threading
import shutil
-import os
-import sys
+import threading
+import time
+
import numpy as np
-from typing import List, Dict, Any
-from concurrent.futures import ThreadPoolExecutor
-from openviking.storage.vectordb.collection.local_collection import get_or_create_local_collection
from openviking.storage.vectordb.collection.collection import Collection
+from openviking.storage.vectordb.collection.local_collection import get_or_create_local_collection
# --- Configuration ---
DEFAULT_DIM = 128
@@ -18,6 +16,7 @@ DEFAULT_DB_PATH = "./benchmark_stress_db"
CATEGORIES = ["news", "sports", "finance", "tech", "entertainment"]
TAGS = ["hot", "new", "archived", "premium", "public"]
+
class StressStats:
def __init__(self):
self.lock = threading.Lock()
@@ -48,75 +47,91 @@ class StressStats:
with self.lock:
duration = time.time() - self.start_time
print(f"\n--- Stress Test Report (Duration: {duration:.2f}s) ---")
- print(f"Insert: {self.insert_count} ops, {self.insert_count/duration:.2f} OPS, Avg Latency: {np.mean(self.insert_latency) if self.insert_latency else 0:.4f}s")
- print(f"Search: {self.search_count} ops, {self.search_count/duration:.2f} OPS, Avg Latency: {np.mean(self.search_latency) if self.search_latency else 0:.4f}s")
- print(f"Delete: {self.delete_count} ops, {self.delete_count/duration:.2f} OPS, Avg Latency: {np.mean(self.delete_latency) if self.delete_latency else 0:.4f}s")
+ print(
+ f"Insert: {self.insert_count} ops, {self.insert_count / duration:.2f} OPS, Avg Latency: {np.mean(self.insert_latency) if self.insert_latency else 0:.4f}s"
+ )
+ print(
+ f"Search: {self.search_count} ops, {self.search_count / duration:.2f} OPS, Avg Latency: {np.mean(self.search_latency) if self.search_latency else 0:.4f}s"
+ )
+ print(
+ f"Delete: {self.delete_count} ops, {self.delete_count / duration:.2f} OPS, Avg Latency: {np.mean(self.delete_latency) if self.delete_latency else 0:.4f}s"
+ )
print(f"Total Ops: {self.insert_count + self.search_count + self.delete_count}")
print("------------------------------------------------")
+
def generate_random_vector(dim):
return [random.random() for _ in range(dim)]
+
def generate_random_sparse_vector():
# Random sparse vector: few random terms with weights
- terms = ["term" + str(i) for i in range(100)] # Vocabulary of 100 terms
+ terms = ["term" + str(i) for i in range(100)] # Vocabulary of 100 terms
num_terms = random.randint(1, 10)
selected = random.sample(terms, num_terms)
return {term: random.random() for term in selected}
+
def setup_collection(path: str, dim: int, enable_sparse: bool):
if os.path.exists(path):
shutil.rmtree(path)
-
+
fields = [
{"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},
{"FieldName": "vector", "FieldType": "vector", "Dim": dim},
{"FieldName": "category", "FieldType": "string"},
{"FieldName": "score", "FieldType": "float32"},
{"FieldName": "is_active", "FieldType": "bool"},
- {"FieldName": "tags", "FieldType": "list<string>"}
+ {"FieldName": "tags", "FieldType": "list<string>"},
]
-
+
if enable_sparse:
fields.append({"FieldName": "sparse_vec", "FieldType": "sparse_vector"})
meta_data = {
"CollectionName": "stress_collection",
"Description": "A collection for violent stress testing",
- "Fields": fields
+ "Fields": fields,
}
-
+
col = get_or_create_local_collection(meta_data=meta_data, path=path)
-
+
# Create multiple indexes to stress the index manager
vector_index_config = {
- "IndexType": "flat_hybrid" if enable_sparse else "flat",
+ "IndexType": "flat_hybrid" if enable_sparse else "flat",
"Distance": "l2",
# "FieldName": "vector" # FieldName is not in Pydantic schema for VectorIndexConfig
}
if enable_sparse:
vector_index_config["EnableSparse"] = True
- col.create_index("idx_vector", {
- "IndexName": "idx_vector",
- "VectorIndex": vector_index_config
- })
-
+ col.create_index("idx_vector", {"IndexName": "idx_vector", "VectorIndex": vector_index_config})
+
# col.create_index("idx_category", {
# "IndexName": "idx_category",
- # "ScalarIndex": ["category"]
+ # "ScalarIndex": ["category"]
# })
-
+
return col
-def worker_insert(col: Collection, stats: StressStats, start_id: int, count: int, batch_size: int, dim: int, enable_sparse: bool, stop_event: threading.Event = None):
+
+def worker_insert(
+ col: Collection,
+ stats: StressStats,
+ start_id: int,
+ count: int,
+ batch_size: int,
+ dim: int,
+ enable_sparse: bool,
+ stop_event: threading.Event = None,
+):
current_id = start_id
end_id = start_id + count
-
+
while current_id < end_id:
if stop_event and stop_event.is_set():
break
-
+
batch_data = []
real_batch_size = min(batch_size, end_id - current_id)
for i in range(real_batch_size):
@@ -126,65 +141,91 @@ def worker_insert(col: Collection, stats: StressStats, start_id: int, count: int
"category": random.choice(CATEGORIES),
"score": random.random(),
"is_active": random.choice([True, False]),
- "tags": random.sample(TAGS, k=random.randint(1, 3))
+ "tags": random.sample(TAGS, k=random.randint(1, 3)),
}
if enable_sparse:
item["sparse_vec"] = generate_random_sparse_vector()
batch_data.append(item)
-
+
t0 = time.time()
try:
col.upsert_data(batch_data)
stats.record_insert(time.time() - t0)
except Exception as e:
print(f"[Insert] Error: {e}")
-
+
current_id += real_batch_size
# time.sleep(0.01) # Small sleep to prevent total lock starvation if any
-def worker_search(col: Collection, stats: StressStats, duration: int, dim: int, max_id: int, enable_sparse: bool, stop_event: threading.Event = None):
+
+def worker_search(
+ col: Collection,
+ stats: StressStats,
+ duration: int,
+ dim: int,
+ max_id: int,
+ enable_sparse: bool,
+ stop_event: threading.Event = None,
+):
end_time = time.time() + duration
while time.time() < end_time:
if stop_event and stop_event.is_set():
break
-
+
t0 = time.time()
try:
# 1. Pure Vector Search
if enable_sparse:
sparse_q = generate_random_sparse_vector()
- col.search_by_vector("idx_vector", generate_random_vector(dim), limit=10, sparse_vector=sparse_q)
+ col.search_by_vector(
+ "idx_vector", generate_random_vector(dim), limit=10, sparse_vector=sparse_q
+ )
else:
col.search_by_vector("idx_vector", generate_random_vector(dim), limit=10)
-
+
# 2. Filtered Vector Search (Complex)
filter_cond = {
"category": {"in": [random.choice(CATEGORIES)]},
"score": {"gt": 0.5},
- "is_active": {"eq": True}
+ "is_active": {"eq": True},
}
if enable_sparse:
sparse_q = generate_random_sparse_vector()
- col.search_by_vector("idx_vector", generate_random_vector(dim), limit=10, filters=filter_cond, sparse_vector=sparse_q)
+ col.search_by_vector(
+ "idx_vector",
+ generate_random_vector(dim),
+ limit=10,
+ filters=filter_cond,
+ sparse_vector=sparse_q,
+ )
else:
- col.search_by_vector("idx_vector", generate_random_vector(dim), limit=10, filters=filter_cond)
-
+ col.search_by_vector(
+ "idx_vector", generate_random_vector(dim), limit=10, filters=filter_cond
+ )
+
# 3. Fetch Data
random_ids = [random.randint(0, max_id) for _ in range(5)]
col.fetch_data(random_ids)
-
+
stats.record_search(time.time() - t0)
except Exception as e:
print(f"[Search] Error: {e}")
-
+
# time.sleep(0.005)
-def worker_delete(col: Collection, stats: StressStats, duration: int, max_id: int, stop_event: threading.Event = None):
+
+def worker_delete(
+ col: Collection,
+ stats: StressStats,
+ duration: int,
+ max_id: int,
+ stop_event: threading.Event = None,
+):
end_time = time.time() + duration
while time.time() < end_time:
if stop_event and stop_event.is_set():
break
-
+
t0 = time.time()
try:
# Randomly delete a small batch
@@ -193,8 +234,9 @@ def worker_delete(col: Collection, stats: StressStats, duration: int, max_id: in
stats.record_delete(time.time() - t0)
except Exception as e:
print(f"[Delete] Error: {e}")
-
- time.sleep(0.1) # Delete less frequently than insert/search
+
+ time.sleep(0.1) # Delete less frequently than insert/search
+
def run_stress_test():
parser = argparse.ArgumentParser(description="Violent Vectordb Stress Test")
@@ -206,58 +248,93 @@ def run_stress_test():
parser.add_argument("--search_threads", type=int, default=8, help="Number of search threads")
parser.add_argument("--delete_threads", type=int, default=2, help="Number of delete threads")
parser.add_argument("--enable_sparse", action="store_true", help="Enable sparse vector support")
-
+
args = parser.parse_args()
-
- print(f"=== Starting Stress Test (Dim={args.dim}, Duration={args.duration}s, Sparse={args.enable_sparse}) ===")
+
+ print(
+ f"=== Starting Stress Test (Dim={args.dim}, Duration={args.duration}s, Sparse={args.enable_sparse}) ==="
+ )
print(f"DB Path: {args.path}")
-
+
col = setup_collection(args.path, args.dim, args.enable_sparse)
stats = StressStats()
-
+
# Preload data
print(f"Preloading {args.initial_data} items...")
- worker_insert(col, stats, 0, args.initial_data, batch_size=100, dim=args.dim, enable_sparse=args.enable_sparse)
+ worker_insert(
+ col,
+ stats,
+ 0,
+ args.initial_data,
+ batch_size=100,
+ dim=args.dim,
+ enable_sparse=args.enable_sparse,
+ )
print("Preload complete.")
-
+
# Reset stats for the actual stress phase
- stats = StressStats()
-
+ stats = StressStats()
+
# Define ID ranges for inserts to avoid massive collisions, though collisions are also fun for testing
# Let's make inserts append new data
start_id_base = args.initial_data
- items_per_thread = 1000000 # Large enough to keep running
-
+ items_per_thread = 1000000 # Large enough to keep running
+
threads = []
-
+
# Stop Event
stop_event = threading.Event()
-
+
# Start Insert Threads
for i in range(args.insert_threads):
t_start = start_id_base + i * items_per_thread
- t = threading.Thread(target=worker_insert, args=(col, stats, t_start, items_per_thread, 50, args.dim, args.enable_sparse, stop_event))
+ t = threading.Thread(
+ target=worker_insert,
+ args=(
+ col,
+ stats,
+ t_start,
+ items_per_thread,
+ 50,
+ args.dim,
+ args.enable_sparse,
+ stop_event,
+ ),
+ )
t.daemon = True
t.start()
threads.append(t)
-
+
# Start Search Threads
- # They will query random IDs up to current approximate max.
+ # They will query random IDs up to current approximate max.
# We estimate max_id conservatively to avoid too many misses, but misses are okay.
estimated_max_id = start_id_base + (items_per_thread * args.insert_threads)
- for i in range(args.search_threads):
- t = threading.Thread(target=worker_search, args=(col, stats, args.duration, args.dim, estimated_max_id, args.enable_sparse, stop_event))
+ for _ in range(args.search_threads):
+ t = threading.Thread(
+ target=worker_search,
+ args=(
+ col,
+ stats,
+ args.duration,
+ args.dim,
+ estimated_max_id,
+ args.enable_sparse,
+ stop_event,
+ ),
+ )
t.daemon = True
t.start()
threads.append(t)
-
+
# Start Delete Threads
- for i in range(args.delete_threads):
- t = threading.Thread(target=worker_delete, args=(col, stats, args.duration, estimated_max_id, stop_event))
+ for _ in range(args.delete_threads):
+ t = threading.Thread(
+ target=worker_delete, args=(col, stats, args.duration, estimated_max_id, stop_event)
+ )
t.daemon = True
t.start()
threads.append(t)
-
+
# Monitor loop
start_time = time.time()
try:
@@ -267,17 +344,17 @@ def run_stress_test():
break
except KeyboardInterrupt:
print("\nInterrupted by user")
-
+
print("\nStopping threads...")
stop_event.set()
-
+
# Wait for all threads to finish
for t in threads:
t.join(timeout=2.0)
-
+
# Threads are daemon, will die when main exits, but let's give a quick report first
stats.report()
-
+
# Cleanup
print("Cleaning up...")
col.close()
@@ -285,5 +362,6 @@ def run_stress_test():
shutil.rmtree(args.path)
print("Done.")
+
if __name__ == "__main__":
run_stress_test()
diff --git a/tests/vectordb/test_bytes_row.py b/tests/vectordb/test_bytes_row.py
index 1bd8b60cc..31b32b16f 100644
--- a/tests/vectordb/test_bytes_row.py
+++ b/tests/vectordb/test_bytes_row.py
@@ -3,6 +3,7 @@
import unittest
from dataclasses import dataclass, field
from typing import List
+
from openviking.storage.vectordb.store.serializable import serializable
@@ -16,12 +17,7 @@ class TestBytesRow(unittest.TestCase):
active: bool = False
name: str = ""
- data = BasicData(
- id=1234567890,
- score=0.95,
- active=True,
- name="viking_db"
- )
+ data = BasicData(id=1234567890, score=0.95, active=True, name="viking_db")
# Serialize
serialized = data.serialize()
@@ -50,9 +46,7 @@ class TestBytesRow(unittest.TestCase):
counts: List[int] = field(default_factory=list)
data = ListData(
- tags=["AI", "Vector", "Search"],
- embedding=[0.1, 0.2, 0.3, 0.4],
- counts=[1, 10, 100]
+ tags=["AI", "Vector", "Search"], embedding=[0.1, 0.2, 0.3, 0.4], counts=[1, 10, 100]
)
serialized = data.serialize()
@@ -97,11 +91,12 @@ class TestBytesRow(unittest.TestCase):
class BinaryData:
raw: bytes = b""
- blob = b"\x00\x01\x02\xFF\xFE"
+ blob = b"\x00\x01\x02\xff\xfe"
data = BinaryData(raw=blob)
serialized = data.serialize()
val = BinaryData.bytes_row.deserialize_field(serialized, "raw")
self.assertEqual(val, blob)
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/vectordb/test_collection_large_scale.py b/tests/vectordb/test_collection_large_scale.py
index 2ac8af9e5..85cac5c3e 100644
--- a/tests/vectordb/test_collection_large_scale.py
+++ b/tests/vectordb/test_collection_large_scale.py
@@ -5,13 +5,11 @@ Large scale scenario tests - Collection stress tests and performance tests
Tests for large data volumes, high-dimensional vectors, complex queries, etc.
"""
-import unittest
-import shutil
-import os
+import gc
import random
+import shutil
import time
-import gc
-from typing import List, Dict, Any
+import unittest
from openviking.storage.vectordb.collection.local_collection import get_or_create_local_collection
@@ -68,7 +66,6 @@ class TestLargeScaleScenarios(unittest.TestCase):
# Prepare 10K data
print("Preparing 10,000 records...")
random.seed(42)
- data_list = []
batch_size = 1000
total_records = 10000
@@ -79,17 +76,21 @@ class TestLargeScaleScenarios(unittest.TestCase):
batch_data = []
for i in range(batch_size):
record_id = batch_idx * batch_size + i + 1
- batch_data.append({
- "id": record_id,
- "vector": [random.gauss(0, 1) for _ in range(128)],
- "category": f"cat_{record_id % 20}",
- "score": random.uniform(0, 100),
- "timestamp": int(time.time()) + record_id,
- })
+ batch_data.append(
+ {
+ "id": record_id,
+ "vector": [random.gauss(0, 1) for _ in range(128)],
+ "category": f"cat_{record_id % 20}",
+ "score": random.uniform(0, 100),
+ "timestamp": int(time.time()) + record_id,
+ }
+ )
result = collection.upsert_data(batch_data)
self.assertEqual(len(result.ids), batch_size)
- print(f" Batch {batch_idx + 1}/{total_records // batch_size} inserted ({len(result.ids)} records)")
+ print(
+ f" Batch {batch_idx + 1}/{total_records // batch_size} inserted ({len(result.ids)} records)"
+ )
insert_time = time.time() - start_time
print(f"βœ“ Inserted {total_records} records in {insert_time:.2f}s")
@@ -104,7 +105,7 @@ class TestLargeScaleScenarios(unittest.TestCase):
"IndexName": "idx_10k",
"VectorIndex": {"IndexType": "flat", "Distance": "ip"},
"ScalarIndex": ["category", "score", "timestamp"],
- }
+ },
)
index_time = time.time() - index_start
print(f"βœ“ Index created in {index_time:.2f}s")
@@ -114,7 +115,7 @@ class TestLargeScaleScenarios(unittest.TestCase):
query_vec = [random.gauss(0, 1) for _ in range(128)]
search_times = []
- for i in range(10):
+ for _ in range(10):
search_start = time.time()
result = collection.search_by_vector("idx_10k", dense_vector=query_vec, limit=100)
search_time = time.time() - search_start
@@ -122,7 +123,7 @@ class TestLargeScaleScenarios(unittest.TestCase):
self.assertEqual(len(result.data), 100)
avg_search_time = sum(search_times) / len(search_times)
- print(f"βœ“ Average search time: {avg_search_time*1000:.2f}ms (10 queries, top-100)")
+ print(f"βœ“ Average search time: {avg_search_time * 1000:.2f}ms (10 queries, top-100)")
# Test filtered search
print("Testing filtered search...")
@@ -131,10 +132,12 @@ class TestLargeScaleScenarios(unittest.TestCase):
"idx_10k",
dense_vector=query_vec,
limit=50,
- filters={"op": "range", "field": "score", "gte": 50.0}
+ filters={"op": "range", "field": "score", "gte": 50.0},
)
filter_time = time.time() - filter_start
- print(f"βœ“ Filtered search completed in {filter_time*1000:.2f}ms (returned {len(result.data)} results)")
+ print(
+ f"βœ“ Filtered search completed in {filter_time * 1000:.2f}ms (returned {len(result.data)} results)"
+ )
def test_02_large_batch_insert_50k(self):
"""Test extra large batch insert - 50,000 records"""
@@ -164,11 +167,13 @@ class TestLargeScaleScenarios(unittest.TestCase):
batch_data = []
for i in range(batch_size):
record_id = batch_idx * batch_size + i + 1
- batch_data.append({
- "id": record_id,
- "vector": [random.uniform(-1, 1) for _ in range(256)],
- "group": f"group_{record_id % 50}",
- })
+ batch_data.append(
+ {
+ "id": record_id,
+ "vector": [random.uniform(-1, 1) for _ in range(256)],
+ "group": f"group_{record_id % 50}",
+ }
+ )
collection.upsert_data(batch_data)
if (batch_idx + 1) % 5 == 0:
@@ -187,7 +192,7 @@ class TestLargeScaleScenarios(unittest.TestCase):
"IndexName": "idx_50k",
"VectorIndex": {"IndexType": "flat", "Distance": "l2"},
"ScalarIndex": ["group"],
- }
+ },
)
index_time = time.time() - index_start
print(f"βœ“ Index created in {index_time:.2f}s")
@@ -230,11 +235,13 @@ class TestLargeScaleScenarios(unittest.TestCase):
record_id = batch_idx * batch_size + i + 1
# Generate high-dimensional vector
vector = [random.gauss(0, 1) for _ in range(1024)]
- batch_data.append({
- "id": record_id,
- "vector": vector,
- "label": f"label_{record_id % 10}",
- })
+ batch_data.append(
+ {
+ "id": record_id,
+ "vector": vector,
+ "label": f"label_{record_id % 10}",
+ }
+ )
collection.upsert_data(batch_data)
if (batch_idx + 1) % 4 == 0:
@@ -250,7 +257,7 @@ class TestLargeScaleScenarios(unittest.TestCase):
{
"IndexName": "idx_high_dim",
"VectorIndex": {"IndexType": "flat", "Distance": "cosine"},
- }
+ },
)
query_vec = [random.gauss(0, 1) for _ in range(1024)]
@@ -259,7 +266,7 @@ class TestLargeScaleScenarios(unittest.TestCase):
search_time = time.time() - search_start
self.assertEqual(len(result.data), 50)
- print(f"βœ“ High-dim search completed in {search_time*1000:.2f}ms (top-50)")
+ print(f"βœ“ High-dim search completed in {search_time * 1000:.2f}ms (top-50)")
def test_04_massive_updates(self):
"""Test massive updates - 10,000 records batch update"""
@@ -295,7 +302,7 @@ class TestLargeScaleScenarios(unittest.TestCase):
insert_start = time.time()
batch_size = 1000
for i in range(0, len(data_list), batch_size):
- collection.upsert_data(data_list[i:i+batch_size])
+ collection.upsert_data(data_list[i : i + batch_size])
insert_time = time.time() - insert_start
print(f"βœ“ Initial insert completed in {insert_time:.2f}s")
@@ -306,7 +313,7 @@ class TestLargeScaleScenarios(unittest.TestCase):
"IndexName": "idx_update",
"VectorIndex": {"IndexType": "flat"},
"ScalarIndex": ["version"],
- }
+ },
)
# Batch update
@@ -322,7 +329,7 @@ class TestLargeScaleScenarios(unittest.TestCase):
update_start = time.time()
for i in range(0, len(update_data), batch_size):
- collection.upsert_data(update_data[i:i+batch_size])
+ collection.upsert_data(update_data[i : i + batch_size])
if (i + batch_size) % 2000 == 0:
print(f" Updated {i + batch_size} / {total_records} records")
update_time = time.time() - update_start
@@ -379,7 +386,7 @@ class TestLargeScaleScenarios(unittest.TestCase):
delete_start = time.time()
for i in range(0, len(delete_ids), delete_batch_size):
- batch_ids = delete_ids[i:i+delete_batch_size]
+ batch_ids = delete_ids[i : i + delete_batch_size]
collection.delete_data(batch_ids)
if (i + delete_batch_size) % 5000 == 0:
print(f" Deleted {i + delete_batch_size} / 25000 records")
@@ -391,7 +398,7 @@ class TestLargeScaleScenarios(unittest.TestCase):
# Verify deletion - Method 1: fetch_data
fetch_result = collection.fetch_data([1, 100, 25000, 25001, 50000])
self.assertEqual(len(fetch_result.items), 2) # Only 25001 and 50000 exist
- self.assertEqual(set([item.id for item in fetch_result.items]), {25001, 50000})
+ self.assertEqual({item.id for item in fetch_result.items}, {25001, 50000})
print("βœ“ Delete verification passed (fetch_data)")
# Verify deletion - Method 2: verify actual retrievable record count through search
@@ -399,20 +406,24 @@ class TestLargeScaleScenarios(unittest.TestCase):
search_result = collection.search_by_vector(
"idx",
dense_vector=[random.random() for _ in range(32)],
- limit=30000 # Request more than remaining count
+ limit=30000, # Request more than remaining count
)
actual_count = len(search_result.data)
print(f" Search returned {actual_count} records (expected ~25000)")
# Index may have delayed updates, so we allow some margin
# But should at least be less than original count
- self.assertLess(actual_count, 30000, "Search should return less than 30000 records after deletion")
+ self.assertLess(
+ actual_count, 30000, "Search should return less than 30000 records after deletion"
+ )
# Verify deleted records are not in search results
search_ids = {item.id for item in search_result.data}
deleted_samples = [1, 100, 1000, 10000, 25000]
for deleted_id in deleted_samples:
- self.assertNotIn(deleted_id, search_ids, f"Deleted ID {deleted_id} should not be in search results")
+ self.assertNotIn(
+ deleted_id, search_ids, f"Deleted ID {deleted_id} should not be in search results"
+ )
print(f"βœ“ Deletion verified through search: {actual_count} records remain")
# Verify deletion - Method 3: aggregate_data to count remaining data
@@ -420,7 +431,11 @@ class TestLargeScaleScenarios(unittest.TestCase):
agg_result = collection.aggregate_data("idx", op="count")
agg_count = agg_result.agg.get("_total", 0)
print(f" Aggregate count: {agg_count}")
- self.assertEqual(agg_count, 25000, f"Expected 25000 records after deletion, but aggregate_data returned {agg_count}")
+ self.assertEqual(
+ agg_count,
+ 25000,
+ f"Expected 25000 records after deletion, but aggregate_data returned {agg_count}",
+ )
print("βœ“ Aggregate count verification passed")
def test_06_complex_multi_filter_large_scale(self):
@@ -456,14 +471,16 @@ class TestLargeScaleScenarios(unittest.TestCase):
batch_data = []
for i in range(batch_size):
record_id = batch_idx * batch_size + i + 1
- batch_data.append({
- "id": record_id,
- "vector": [random.gauss(0, 1) for _ in range(128)],
- "category": random.choice(categories),
- "priority": random.randint(1, 10),
- "score": random.uniform(0, 100),
- "status": random.choice(statuses),
- })
+ batch_data.append(
+ {
+ "id": record_id,
+ "vector": [random.gauss(0, 1) for _ in range(128)],
+ "category": random.choice(categories),
+ "priority": random.randint(1, 10),
+ "score": random.uniform(0, 100),
+ "status": random.choice(statuses),
+ }
+ )
collection.upsert_data(batch_data)
print(f"βœ“ Inserted {total_records} records")
@@ -475,7 +492,7 @@ class TestLargeScaleScenarios(unittest.TestCase):
"IndexName": "idx_complex",
"VectorIndex": {"IndexType": "flat", "Distance": "ip"},
"ScalarIndex": ["category", "priority", "score", "status"],
- }
+ },
)
# Test complex filter queries
@@ -489,14 +506,18 @@ class TestLargeScaleScenarios(unittest.TestCase):
{"op": "range", "field": "priority", "gte": 7},
{"op": "range", "field": "score", "gt": 50.0},
{"op": "must", "field": "status", "conds": ["active"]},
- ]
+ ],
}
query_vec = [random.gauss(0, 1) for _ in range(128)]
search_start = time.time()
- result1 = collection.search_by_vector("idx_complex", dense_vector=query_vec, limit=100, filters=filter1)
+ result1 = collection.search_by_vector(
+ "idx_complex", dense_vector=query_vec, limit=100, filters=filter1
+ )
search_time1 = time.time() - search_start
- print(f" Query 1 (4 conditions): {len(result1.data)} results in {search_time1*1000:.2f}ms")
+ print(
+ f" Query 1 (4 conditions): {len(result1.data)} results in {search_time1 * 1000:.2f}ms"
+ )
# Query 2: (category="C" OR category="D") AND priority IN [3,5,7] AND status != "inactive"
filter2 = {
@@ -505,13 +526,17 @@ class TestLargeScaleScenarios(unittest.TestCase):
{"op": "must", "field": "category", "conds": ["C", "D"]},
{"op": "must", "field": "priority", "conds": [3, 5, 7]},
{"op": "must_not", "field": "status", "conds": ["inactive"]},
- ]
+ ],
}
search_start = time.time()
- result2 = collection.search_by_vector("idx_complex", dense_vector=query_vec, limit=100, filters=filter2)
+ result2 = collection.search_by_vector(
+ "idx_complex", dense_vector=query_vec, limit=100, filters=filter2
+ )
search_time2 = time.time() - search_start
- print(f" Query 2 (3 conditions): {len(result2.data)} results in {search_time2*1000:.2f}ms")
+ print(
+ f" Query 2 (3 conditions): {len(result2.data)} results in {search_time2 * 1000:.2f}ms"
+ )
# Query 3: Range query + sort
search_start = time.time()
@@ -520,10 +545,12 @@ class TestLargeScaleScenarios(unittest.TestCase):
field="score",
order="desc",
limit=500,
- filters={"op": "range", "field": "priority", "gte": 5, "lte": 8}
+ filters={"op": "range", "field": "priority", "gte": 5, "lte": 8},
)
search_time3 = time.time() - search_start
- print(f" Query 3 (scalar sort): {len(result3.data)} results in {search_time3*1000:.2f}ms")
+ print(
+ f" Query 3 (scalar sort): {len(result3.data)} results in {search_time3 * 1000:.2f}ms"
+ )
print("βœ“ Complex multi-filter queries completed")
@@ -560,13 +587,15 @@ class TestLargeScaleScenarios(unittest.TestCase):
batch_data = []
for i in range(batch_size):
record_id = batch_idx * batch_size + i + 1
- batch_data.append({
- "id": record_id,
- "vector": [random.random() for _ in range(64)],
- "country": random.choice(countries),
- "city": random.choice(cities),
- "product": random.choice(products),
- })
+ batch_data.append(
+ {
+ "id": record_id,
+ "vector": [random.random() for _ in range(64)],
+ "country": random.choice(countries),
+ "city": random.choice(cities),
+ "product": random.choice(products),
+ }
+ )
collection.upsert_data(batch_data)
print(f"βœ“ Inserted {total_records} records")
@@ -578,7 +607,7 @@ class TestLargeScaleScenarios(unittest.TestCase):
"IndexName": "idx_agg",
"VectorIndex": {"IndexType": "flat"},
"ScalarIndex": ["country", "city", "product"],
- }
+ },
)
# Test aggregations with different cardinalities
@@ -588,19 +617,25 @@ class TestLargeScaleScenarios(unittest.TestCase):
agg_start = time.time()
result1 = collection.aggregate_data("idx_agg", op="count", field="country")
agg_time1 = time.time() - agg_start
- print(f" Country aggregation (50 groups): {len(result1.agg)} groups in {agg_time1*1000:.2f}ms")
+ print(
+ f" Country aggregation (50 groups): {len(result1.agg)} groups in {agg_time1 * 1000:.2f}ms"
+ )
# Medium cardinality aggregation (100 groups)
agg_start = time.time()
result2 = collection.aggregate_data("idx_agg", op="count", field="product")
agg_time2 = time.time() - agg_start
- print(f" Product aggregation (100 groups): {len(result2.agg)} groups in {agg_time2*1000:.2f}ms")
+ print(
+ f" Product aggregation (100 groups): {len(result2.agg)} groups in {agg_time2 * 1000:.2f}ms"
+ )
# High cardinality aggregation (200 groups)
agg_start = time.time()
result3 = collection.aggregate_data("idx_agg", op="count", field="city")
agg_time3 = time.time() - agg_start
- print(f" City aggregation (200 groups): {len(result3.agg)} groups in {agg_time3*1000:.2f}ms")
+ print(
+ f" City aggregation (200 groups): {len(result3.agg)} groups in {agg_time3 * 1000:.2f}ms"
+ )
# Filtered aggregation
agg_start = time.time()
@@ -608,10 +643,14 @@ class TestLargeScaleScenarios(unittest.TestCase):
"idx_agg",
op="count",
field="product",
- filters={"op": "must", "field": "country", "conds": ["country_0", "country_1", "country_2"]}
+ filters={
+ "op": "must",
+ "field": "country",
+ "conds": ["country_0", "country_1", "country_2"],
+ },
)
agg_time4 = time.time() - agg_start
- print(f" Filtered aggregation: {len(result4.agg)} groups in {agg_time4*1000:.2f}ms")
+ print(f" Filtered aggregation: {len(result4.agg)} groups in {agg_time4 * 1000:.2f}ms")
print("βœ“ Large scale aggregation tests completed")
@@ -641,11 +680,13 @@ class TestLargeScaleScenarios(unittest.TestCase):
batch_data = []
for i in range(batch_size):
record_id = batch_idx * batch_size + i + 1
- batch_data.append({
- "id": record_id,
- "vector": [random.uniform(-1, 1) for _ in range(256)],
- "metadata": f"metadata_{record_id}",
- })
+ batch_data.append(
+ {
+ "id": record_id,
+ "vector": [random.uniform(-1, 1) for _ in range(256)],
+ "metadata": f"metadata_{record_id}",
+ }
+ )
collection1.upsert_data(batch_data)
write_time = time.time() - write_start
@@ -659,12 +700,14 @@ class TestLargeScaleScenarios(unittest.TestCase):
"IndexName": "idx_persist",
"VectorIndex": {"IndexType": "flat", "Distance": "l2"},
"ScalarIndex": ["metadata"],
- }
+ },
)
# Execute search and record results
query_vec = [random.uniform(-1, 1) for _ in range(256)]
- result_before = collection1.search_by_vector("idx_persist", dense_vector=query_vec, limit=50)
+ result_before = collection1.search_by_vector(
+ "idx_persist", dense_vector=query_vec, limit=50
+ )
ids_before = [item.id for item in result_before.data]
# Close
@@ -677,9 +720,7 @@ class TestLargeScaleScenarios(unittest.TestCase):
# Phase 2: Reload
print("\nPhase 2: Reloading from disk...")
reload_start = time.time()
- collection2 = self.register_collection(
- get_or_create_local_collection(path=TEST_DB_PATH)
- )
+ collection2 = self.register_collection(get_or_create_local_collection(path=TEST_DB_PATH))
reload_time = time.time() - reload_start
print(f"βœ“ Collection reloaded in {reload_time:.2f}s")
@@ -702,7 +743,6 @@ class TestLargeScaleScenarios(unittest.TestCase):
self.assertEqual(agg_result.agg["_total"], total_records)
print(f"βœ“ Total count verified: {agg_result.agg['_total']}")
-
def test_09_concurrent_operations_simulation(self):
"""Simulate concurrent operations scenario (serial simulation)"""
print("\n=== Test 9: Concurrent Operations Simulation ===")
@@ -739,7 +779,7 @@ class TestLargeScaleScenarios(unittest.TestCase):
"IndexName": "idx_concurrent",
"VectorIndex": {"IndexType": "flat"},
"ScalarIndex": ["category"],
- }
+ },
)
print("βœ“ Initial setup completed (5000 records)")
@@ -748,30 +788,40 @@ class TestLargeScaleScenarios(unittest.TestCase):
operations = []
# 100 searches
- for i in range(100):
+ for _ in range(100):
query_vec = [random.random() for _ in range(64)]
operations.append(("search", query_vec))
# 50 inserts
for i in range(50):
new_id = 5001 + i
- operations.append(("insert", {
- "id": new_id,
- "vector": [random.random() for _ in range(64)],
- "category": f"cat_{new_id % 10}",
- }))
+ operations.append(
+ (
+ "insert",
+ {
+ "id": new_id,
+ "vector": [random.random() for _ in range(64)],
+ "category": f"cat_{new_id % 10}",
+ },
+ )
+ )
# 30 updates
- for i in range(30):
+ for _ in range(30):
update_id = random.randint(1, 5000)
- operations.append(("update", {
- "id": update_id,
- "vector": [random.random() for _ in range(64)],
- "category": f"cat_updated",
- }))
+ operations.append(
+ (
+ "update",
+ {
+ "id": update_id,
+ "vector": [random.random() for _ in range(64)],
+ "category": "cat_updated",
+ },
+ )
+ )
# 20 deletes
- for i in range(20):
+ for _ in range(20):
delete_id = random.randint(1, 1000)
operations.append(("delete", delete_id))
@@ -786,7 +836,7 @@ class TestLargeScaleScenarios(unittest.TestCase):
for op_type, op_data in operations:
if op_type == "search":
- result = collection.search_by_vector("idx_concurrent", dense_vector=op_data, limit=10)
+ collection.search_by_vector("idx_concurrent", dense_vector=op_data, limit=10)
search_count += 1
elif op_type == "insert":
collection.upsert_data([op_data])
@@ -837,7 +887,9 @@ class TestLargeScaleScenarios(unittest.TestCase):
]
collection.upsert_data(batch_data)
- collection.create_index("idx_stress", {"IndexName": "idx_stress", "VectorIndex": {"IndexType": "flat"}})
+ collection.create_index(
+ "idx_stress", {"IndexName": "idx_stress", "VectorIndex": {"IndexType": "flat"}}
+ )
print("βœ“ Initial setup completed")
# Stress test: run for 5 seconds
@@ -849,7 +901,9 @@ class TestLargeScaleScenarios(unittest.TestCase):
while time.time() - start_time < test_duration:
# Randomly select operation type
- op = random.choice(["search", "search", "search", "insert", "update"]) # Higher weight for search
+ op = random.choice(
+ ["search", "search", "search", "insert", "update"]
+ ) # Higher weight for search
if op == "search":
query_vec = [random.gauss(0, 1) for _ in range(128)]
@@ -859,16 +913,24 @@ class TestLargeScaleScenarios(unittest.TestCase):
search_times.append(search_time)
elif op == "insert":
new_id = random.randint(10001, 20000)
- collection.upsert_data([{
- "id": new_id,
- "vector": [random.gauss(0, 1) for _ in range(128)],
- }])
+ collection.upsert_data(
+ [
+ {
+ "id": new_id,
+ "vector": [random.gauss(0, 1) for _ in range(128)],
+ }
+ ]
+ )
elif op == "update":
update_id = random.randint(1, 10000)
- collection.upsert_data([{
- "id": update_id,
- "vector": [random.gauss(0, 1) for _ in range(128)],
- }])
+ collection.upsert_data(
+ [
+ {
+ "id": update_id,
+ "vector": [random.gauss(0, 1) for _ in range(128)],
+ }
+ ]
+ )
operation_count += 1
@@ -881,7 +943,9 @@ class TestLargeScaleScenarios(unittest.TestCase):
avg_search = sum(search_times) / len(search_times)
min_search = min(search_times)
max_search = max(search_times)
- print(f" Search latency: avg={avg_search*1000:.2f}ms, min={min_search*1000:.2f}ms, max={max_search*1000:.2f}ms")
+ print(
+ f" Search latency: avg={avg_search * 1000:.2f}ms, min={min_search * 1000:.2f}ms, max={max_search * 1000:.2f}ms"
+ )
def run_large_scale_tests():
@@ -890,21 +954,21 @@ def run_large_scale_tests():
runner = unittest.TextTestRunner(verbosity=2)
result = runner.run(suite)
- print("\n" + "="*70)
+ print("\n" + "=" * 70)
print("Large Scale Test Summary:")
print(f"Total tests run: {result.testsRun}")
print(f"Successes: {result.testsRun - len(result.failures) - len(result.errors)}")
print(f"Failures: {len(result.failures)}")
print(f"Errors: {len(result.errors)}")
- print("="*70)
+ print("=" * 70)
return result.wasSuccessful()
if __name__ == "__main__":
- print("\n" + "="*70)
+ print("\n" + "=" * 70)
print("OpenViking Collection - Large Scale Scenario Tests")
- print("="*70)
+ print("=" * 70)
success = run_large_scale_tests()
exit(0 if success else 1)
diff --git a/tests/vectordb/test_crash_recovery.py b/tests/vectordb/test_crash_recovery.py
index 940055ea8..88585cad8 100644
--- a/tests/vectordb/test_crash_recovery.py
+++ b/tests/vectordb/test_crash_recovery.py
@@ -3,17 +3,16 @@
import multiprocessing
import os
import shutil
+import sys
import time
import unittest
-import sys
-import random
-from typing import List, Dict, Any
from openviking.storage.vectordb.collection.local_collection import get_or_create_local_collection
DB_PATH_CRASH = "./test_db_crash_recovery"
DB_PATH_ROBUST = "./test_db_robust_crash"
+
def worker_write_and_crash(path, start_id, count, event_ready):
"""
Subprocess function:
@@ -29,35 +28,37 @@ def worker_write_and_crash(path, start_id, count, event_ready):
"Fields": [
{"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},
{"FieldName": "vector", "FieldType": "vector", "Dim": 4},
- {"FieldName": "data", "FieldType": "string"}
- ]
+ {"FieldName": "data", "FieldType": "string"},
+ ],
}
# Force a fresh instance in this process
col = get_or_create_local_collection(meta_data=meta_data, path=path)
-
+
if not col.has_index("idx_crash"):
- col.create_index("idx_crash", {
- "IndexName": "idx_crash",
- "VectorIndex": {"IndexType": "flat", "Distance": "l2"}
- })
-
+ col.create_index(
+ "idx_crash",
+ {"IndexName": "idx_crash", "VectorIndex": {"IndexType": "flat", "Distance": "l2"}},
+ )
+
# Write data
data = []
for i in range(count):
uid = start_id + i
- data.append({
- "id": uid,
- "vector": [0.1] * 4, # Use constant vector for easier search verification
- "data": f"crash_data_{uid}"
- })
-
+ data.append(
+ {
+ "id": uid,
+ "vector": [0.1] * 4, # Use constant vector for easier search verification
+ "data": f"crash_data_{uid}",
+ }
+ )
+
print(f"[Subprocess] Upserting {count} items...")
col.upsert_data(data)
- print(f"[Subprocess] Upsert done. Not closing.")
-
+ print("[Subprocess] Upsert done. Not closing.")
+
# Notify main process that write is done
event_ready.set()
-
+
# Simulate work or wait to be killed
# DO NOT close collection
# We sleep long enough for parent to kill us
@@ -66,6 +67,7 @@ def worker_write_and_crash(path, start_id, count, event_ready):
print(f"[Subprocess] Error: {e}")
sys.exit(1)
+
def setup_robust_collection(path):
"""Helper to setup collection config for robust test"""
meta_data = {
@@ -74,17 +76,18 @@ def setup_robust_collection(path):
{"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},
{"FieldName": "vector", "FieldType": "vector", "Dim": 4},
{"FieldName": "data", "FieldType": "string"},
- {"FieldName": "tag", "FieldType": "string"}
- ]
+ {"FieldName": "tag", "FieldType": "string"},
+ ],
}
col = get_or_create_local_collection(meta_data=meta_data, path=path)
if not col.has_index("idx_main"):
- col.create_index("idx_main", {
- "IndexName": "idx_main",
- "VectorIndex": {"IndexType": "flat", "Distance": "l2"}
- })
+ col.create_index(
+ "idx_main",
+ {"IndexName": "idx_main", "VectorIndex": {"IndexType": "flat", "Distance": "l2"}},
+ )
return col
+
def worker_cycle_1_write(path, event_ready):
"""
Cycle 1: Write 500 items (ID 0-499).
@@ -92,26 +95,22 @@ def worker_cycle_1_write(path, event_ready):
"""
try:
col = setup_robust_collection(path)
-
+
data = []
for i in range(500):
- data.append({
- "id": i,
- "vector": [0.1] * 4,
- "data": f"cycle1_{i}",
- "tag": "c1"
- })
-
+ data.append({"id": i, "vector": [0.1] * 4, "data": f"cycle1_{i}", "tag": "c1"})
+
print("[Cycle 1] Upserting 500 items...")
col.upsert_data(data)
print("[Cycle 1] Upsert done.")
-
+
event_ready.set()
- time.sleep(60) # Wait to be killed
+ time.sleep(60) # Wait to be killed
except Exception as e:
print(f"[Cycle 1] Error: {e}")
sys.exit(1)
+
def worker_cycle_2_mix(path, event_ready):
"""
Cycle 2:
@@ -122,37 +121,33 @@ def worker_cycle_2_mix(path, event_ready):
"""
try:
col = setup_robust_collection(path)
-
+
# Verify Cycle 1 data exists before modification
res = col.fetch_data([0, 499])
if len(res.items) != 2:
print("[Cycle 2] Critical: Cycle 1 data missing on startup!")
sys.exit(1)
-
+
# Delete 0-99
print("[Cycle 2] Deleting 100 items (0-99)...")
col.delete_data(list(range(100)))
-
+
# Upsert 1000-1299
print("[Cycle 2] Upserting 300 items (1000-1299)...")
data = []
for i in range(300):
uid = 1000 + i
- data.append({
- "id": uid,
- "vector": [0.2] * 4,
- "data": f"cycle2_{uid}",
- "tag": "c2"
- })
+ data.append({"id": uid, "vector": [0.2] * 4, "data": f"cycle2_{uid}", "tag": "c2"})
col.upsert_data(data)
print("[Cycle 2] Operations done.")
-
+
event_ready.set()
- time.sleep(60) # Wait to be killed
+ time.sleep(60) # Wait to be killed
except Exception as e:
print(f"[Cycle 2] Error: {e}")
sys.exit(1)
+
class TestCrashRecovery(unittest.TestCase):
def setUp(self):
if os.path.exists(DB_PATH_CRASH):
@@ -172,82 +167,82 @@ class TestCrashRecovery(unittest.TestCase):
(where close() was not called).
"""
print("\n=== Test Simple Crash Recovery ===")
-
+
# Use 'spawn' to ensure clean process state (especially important for RocksDB/LevelDB locks)
- ctx = multiprocessing.get_context('spawn')
+ ctx = multiprocessing.get_context("spawn")
event = ctx.Event()
-
+
data_count = 100
p = ctx.Process(target=worker_write_and_crash, args=(DB_PATH_CRASH, 0, data_count, event))
p.start()
-
+
# Wait for write to complete in subprocess
print("[Main] Waiting for subprocess to write data...")
is_set = event.wait(timeout=5)
self.assertTrue(is_set, "Subprocess timed out writing data")
-
+
# Give it a tiny moment to ensure the OS flush might happen (though we want to test robustness)
# But immediate kill is what we want to test.
# However, Python's LevelDB binding might be sync or async. Usually writes go to OS cache.
- time.sleep(0.5)
-
+ time.sleep(0.5)
+
# KILL the process immediately to simulate crash (no cleanup, no flush)
print("[Main] Terminating subprocess (Simulating Crash)...")
p.terminate()
p.join()
print(f"[Main] Subprocess terminated with exit code: {p.exitcode}")
-
+
# Recover
print("[Main] Recovering collection in main process...")
# Re-opening the collection should trigger recovery logic (PersistentIndex should rebuild/catchup from Store)
col = get_or_create_local_collection(path=DB_PATH_CRASH)
-
+
# 1. Verify Data Persistence (Store)
print("[Main] Verifying data fetch (All 100 items)...")
# Check all IDs
all_ids = list(range(data_count))
res = col.fetch_data(all_ids)
-
+
print(f"[Main] Fetch retrieved {len(res.items)} items. Missing: {len(res.ids_not_exist)}")
-
- self.assertEqual(len(res.items), data_count, f"Should find all {data_count} items in KV Store")
-
+
+ self.assertEqual(
+ len(res.items), data_count, f"Should find all {data_count} items in KV Store"
+ )
+
print("[Main] Data fetch verified.")
-
+
# 2. Verify Index Recovery
- # Since we killed the process, the Index (which might be in memory or partially written)
+ # Since we killed the process, the Index (which might be in memory or partially written)
# needs to be reconstructed from the KV Store delta during initialization.
print("[Main] Verifying vector search (Index Recovery)...")
-
+
# Searching for vector [0.1, 0.1, 0.1, 0.1] should return our data
- search_res = col.search_by_vector(
- "idx_crash",
- dense_vector=[0.1]*4,
- limit=data_count
- )
-
+ search_res = col.search_by_vector("idx_crash", dense_vector=[0.1] * 4, limit=data_count)
+
found_ids = [item.id for item in search_res.data]
print(f"[Main] Search returned {len(found_ids)} items.")
-
+
# We expect all items to be found if index recovery works
- self.assertEqual(len(found_ids), data_count, "Index should contain all items after recovery")
-
+ self.assertEqual(
+ len(found_ids), data_count, "Index should contain all items after recovery"
+ )
+
col.close()
print("[Main] Simple recovery test passed.")
def run_process_and_crash(self, target_func):
- ctx = multiprocessing.get_context('spawn')
+ ctx = multiprocessing.get_context("spawn")
event = ctx.Event()
p = ctx.Process(target=target_func, args=(DB_PATH_ROBUST, event))
p.start()
-
+
# Wait for work done
is_set = event.wait(timeout=5)
self.assertTrue(is_set, "Subprocess timed out")
-
+
# Give a split second for OS buffers (simulate sudden power loss/crash)
time.sleep(0.5)
-
+
print(f"[Main] Crashing process {p.pid}...")
p.terminate()
p.join()
@@ -255,71 +250,72 @@ class TestCrashRecovery(unittest.TestCase):
def test_multi_cycle_crash(self):
print("\n=== Test Robust Multi-Cycle Crash Recovery ===")
-
+
# --- Cycle 1: Write & Crash ---
print("\n--- Starting Cycle 1 ---")
self.run_process_and_crash(worker_cycle_1_write)
-
+
# Verify Cycle 1 Recovery
print("[Main] Verifying Cycle 1 recovery...")
col = setup_robust_collection(DB_PATH_ROBUST)
-
+
# Check counts
- res_search = col.search_by_vector("idx_main", [0.1]*4, limit=1000)
+ res_search = col.search_by_vector("idx_main", [0.1] * 4, limit=1000)
self.assertEqual(len(res_search.data), 500, "Should have 500 items after Cycle 1")
col.close()
-
+
# --- Cycle 2: Mix Ops & Crash ---
print("\n--- Starting Cycle 2 ---")
self.run_process_and_crash(worker_cycle_2_mix)
-
+
# Verify Cycle 2 Recovery
print("[Main] Verifying Cycle 2 recovery...")
col = setup_robust_collection(DB_PATH_ROBUST)
-
+
# 1. Verify Deletions (0-99 should be gone)
print("[Main] Verifying deletions (0-99)...")
res_deleted = col.fetch_data(list(range(10)))
self.assertEqual(len(res_deleted.items), 0, "IDs 0-9 should be deleted")
-
+
# Search check for deleted items
- # IDs 0-99 had vector [0.1]*4.
+ # IDs 0-99 had vector [0.1]*4.
# IDs 100-499 still have [0.1]*4.
# So searching [0.1]*4 should return 400 items that match perfectly (score ~0)
# But vector search returns all items up to limit.
- res_search_old = col.search_by_vector("idx_main", [0.1]*4, limit=1000)
-
+ res_search_old = col.search_by_vector("idx_main", [0.1] * 4, limit=1000)
+
# Verify total count is 700 (400 old + 300 new)
self.assertEqual(len(res_search_old.data), 700, "Total index items should be 700")
# Verify IDs 0-99 are GONE
- found_ids = set(item.id for item in res_search_old.data)
+ found_ids = {item.id for item in res_search_old.data}
for i in range(100):
self.assertNotIn(i, found_ids, f"ID {i} should have been deleted")
-
+
# Verify IDs 100-499 EXIST
for i in range(100, 500):
- self.assertIn(i, found_ids, f"ID {i} should exist")
+ self.assertIn(i, found_ids, f"ID {i} should exist")
# 2. Verify New Inserts (1000-1299 should exist)
print("[Main] Verifying new inserts (1000-1299)...")
res_new = col.fetch_data([1000, 1299])
self.assertEqual(len(res_new.items), 2, "New items 1000 and 1299 should exist")
-
+
# Search check for new items ([0.2]*4)
# Should also return all 700 items, but sorted differently
- res_search_new = col.search_by_vector("idx_main", [0.2]*4, limit=1000)
+ res_search_new = col.search_by_vector("idx_main", [0.2] * 4, limit=1000)
self.assertEqual(len(res_search_new.data), 700, "Should return all 700 items")
-
+
# Verify IDs 1000-1299 EXIST in the result
- found_ids_new = set(item.id for item in res_search_new.data)
+ found_ids_new = {item.id for item in res_search_new.data}
for i in range(1000, 1300):
self.assertIn(i, found_ids_new, f"ID {i} should exist")
-
- print(f"[Main] Total items verified via index: 700")
-
+
+ print("[Main] Total items verified via index: 700")
+
col.close()
print("\n[Main] Robust recovery test passed.")
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/vectordb/test_filter_ops.py b/tests/vectordb/test_filter_ops.py
index 617bde4b0..b57f682e3 100644
--- a/tests/vectordb/test_filter_ops.py
+++ b/tests/vectordb/test_filter_ops.py
@@ -1,12 +1,12 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
-import unittest
+import gc
+import random
import shutil
import time
-import random
-import string
-import gc
+import unittest
+
from openviking.storage.vectordb.collection.local_collection import get_or_create_local_collection
db_path_basic = "./db_test_filters_basic/"
@@ -14,11 +14,14 @@ db_path_complex = "./db_test_filters_complex/"
db_path_lifecycle = "./db_test_filters_lifecycle/"
db_path_scale = "./db_test_filters_scale/"
+
def clean_dir(path):
shutil.rmtree(path, ignore_errors=True)
+
class TestFilterOpsBasic(unittest.TestCase):
"""Basic Filter operator tests"""
+
def setUp(self):
clean_dir(db_path_basic)
self.path = db_path_basic
@@ -46,11 +49,41 @@ class TestFilterOpsBasic(unittest.TestCase):
def _insert_data(self):
data = [
- {"id": 1, "embedding": [1.0, 0, 0, 0], "val_int": 10, "val_float": 1.1, "val_str": "apple"},
- {"id": 2, "embedding": [1.0, 0, 0, 0], "val_int": 20, "val_float": 2.2, "val_str": "banana"},
- {"id": 3, "embedding": [1.0, 0, 0, 0], "val_int": 30, "val_float": 3.3, "val_str": "cherry"},
- {"id": 4, "embedding": [1.0, 0, 0, 0], "val_int": 40, "val_float": 4.4, "val_str": "date"},
- {"id": 5, "embedding": [1.0, 0, 0, 0], "val_int": 50, "val_float": 5.5, "val_str": "elderberry"},
+ {
+ "id": 1,
+ "embedding": [1.0, 0, 0, 0],
+ "val_int": 10,
+ "val_float": 1.1,
+ "val_str": "apple",
+ },
+ {
+ "id": 2,
+ "embedding": [1.0, 0, 0, 0],
+ "val_int": 20,
+ "val_float": 2.2,
+ "val_str": "banana",
+ },
+ {
+ "id": 3,
+ "embedding": [1.0, 0, 0, 0],
+ "val_int": 30,
+ "val_float": 3.3,
+ "val_str": "cherry",
+ },
+ {
+ "id": 4,
+ "embedding": [1.0, 0, 0, 0],
+ "val_int": 40,
+ "val_float": 4.4,
+ "val_str": "date",
+ },
+ {
+ "id": 5,
+ "embedding": [1.0, 0, 0, 0],
+ "val_int": 50,
+ "val_float": 5.5,
+ "val_str": "elderberry",
+ },
]
self.collection.upsert_data(data)
@@ -64,10 +97,7 @@ class TestFilterOpsBasic(unittest.TestCase):
def _search(self, filters):
res = self.collection.search_by_vector(
- "idx_basic",
- dense_vector=[1.0, 0, 0, 0],
- limit=100,
- filters=filters
+ "idx_basic", dense_vector=[1.0, 0, 0, 0], limit=100, filters=filters
)
return sorted([item.id for item in res.data])
@@ -79,11 +109,14 @@ class TestFilterOpsBasic(unittest.TestCase):
# Prefix
self.assertEqual(self._search({"op": "prefix", "field": "val_str", "prefix": "ap"}), [1])
# Contains
- self.assertEqual(self._search({"op": "contains", "field": "val_str", "substring": "er"}), [3, 5]) # chERry, eldERbERry
+ self.assertEqual(
+ self._search({"op": "contains", "field": "val_str", "substring": "er"}), [3, 5]
+ ) # chERry, eldERbERry
class TestFilterOpsComplex(unittest.TestCase):
"""Complex mixed logic tests"""
+
def setUp(self):
clean_dir(db_path_complex)
self.path = db_path_complex
@@ -105,7 +138,10 @@ class TestFilterOpsComplex(unittest.TestCase):
{"FieldName": "category", "FieldType": "string"},
{"FieldName": "tags", "FieldType": "string"},
{"FieldName": "price", "FieldType": "int64"},
- {"FieldName": "rating_int", "FieldType": "int64"}, # Use int64 instead of float32 to avoid potential bug
+ {
+ "FieldName": "rating_int",
+ "FieldType": "int64",
+ }, # Use int64 instead of float32 to avoid potential bug
],
}
return get_or_create_local_collection(meta_data=collection_meta, path=self.path)
@@ -114,16 +150,86 @@ class TestFilterOpsComplex(unittest.TestCase):
# Build some slightly complex data scenarios
# id 1-10
data = [
- {"id": 1, "embedding": [1.0, 0, 0, 0], "category": "electronics", "tags": "mobile,apple,new", "price": 8000, "rating_int": 48},
- {"id": 2, "embedding": [1.0, 0, 0, 0], "category": "electronics", "tags": "mobile,android,sale", "price": 3000, "rating_int": 45},
- {"id": 3, "embedding": [1.0, 0, 0, 0], "category": "electronics", "tags": "laptop,apple,pro", "price": 15000, "rating_int": 49},
- {"id": 4, "embedding": [1.0, 0, 0, 0], "category": "electronics", "tags": "laptop,windows,budget", "price": 4000, "rating_int": 40},
- {"id": 5, "embedding": [1.0, 0, 0, 0], "category": "home", "tags": "furniture,sofa", "price": 2000, "rating_int": 42},
- {"id": 6, "embedding": [1.0, 0, 0, 0], "category": "home", "tags": "kitchen,blender", "price": 300, "rating_int": 38},
- {"id": 7, "embedding": [1.0, 0, 0, 0], "category": "books", "tags": "fiction,sci-fi", "price": 50, "rating_int": 47},
- {"id": 8, "embedding": [1.0, 0, 0, 0], "category": "books", "tags": "fiction,fantasy", "price": 60, "rating_int": 46},
- {"id": 9, "embedding": [1.0, 0, 0, 0], "category": "clothing", "tags": "shirt,summer", "price": 100, "rating_int": 41},
- {"id": 10, "embedding": [1.0, 0, 0, 0], "category": "clothing", "tags": "pants,winter", "price": 200, "rating_int": 43},
+ {
+ "id": 1,
+ "embedding": [1.0, 0, 0, 0],
+ "category": "electronics",
+ "tags": "mobile,apple,new",
+ "price": 8000,
+ "rating_int": 48,
+ },
+ {
+ "id": 2,
+ "embedding": [1.0, 0, 0, 0],
+ "category": "electronics",
+ "tags": "mobile,android,sale",
+ "price": 3000,
+ "rating_int": 45,
+ },
+ {
+ "id": 3,
+ "embedding": [1.0, 0, 0, 0],
+ "category": "electronics",
+ "tags": "laptop,apple,pro",
+ "price": 15000,
+ "rating_int": 49,
+ },
+ {
+ "id": 4,
+ "embedding": [1.0, 0, 0, 0],
+ "category": "electronics",
+ "tags": "laptop,windows,budget",
+ "price": 4000,
+ "rating_int": 40,
+ },
+ {
+ "id": 5,
+ "embedding": [1.0, 0, 0, 0],
+ "category": "home",
+ "tags": "furniture,sofa",
+ "price": 2000,
+ "rating_int": 42,
+ },
+ {
+ "id": 6,
+ "embedding": [1.0, 0, 0, 0],
+ "category": "home",
+ "tags": "kitchen,blender",
+ "price": 300,
+ "rating_int": 38,
+ },
+ {
+ "id": 7,
+ "embedding": [1.0, 0, 0, 0],
+ "category": "books",
+ "tags": "fiction,sci-fi",
+ "price": 50,
+ "rating_int": 47,
+ },
+ {
+ "id": 8,
+ "embedding": [1.0, 0, 0, 0],
+ "category": "books",
+ "tags": "fiction,fantasy",
+ "price": 60,
+ "rating_int": 46,
+ },
+ {
+ "id": 9,
+ "embedding": [1.0, 0, 0, 0],
+ "category": "clothing",
+ "tags": "shirt,summer",
+ "price": 100,
+ "rating_int": 41,
+ },
+ {
+ "id": 10,
+ "embedding": [1.0, 0, 0, 0],
+ "category": "clothing",
+ "tags": "pants,winter",
+ "price": 200,
+ "rating_int": 43,
+ },
]
self.collection.upsert_data(data)
@@ -137,10 +243,7 @@ class TestFilterOpsComplex(unittest.TestCase):
def _search(self, filters):
res = self.collection.search_by_vector(
- "idx_complex",
- dense_vector=[1.0, 0, 0, 0],
- limit=100,
- filters=filters
+ "idx_complex", dense_vector=[1.0, 0, 0, 0], limit=100, filters=filters
)
return sorted([item.id for item in res.data])
@@ -160,10 +263,10 @@ class TestFilterOpsComplex(unittest.TestCase):
"op": "or",
"conds": [
{"op": "contains", "field": "tags", "substring": "apple"},
- {"op": "range", "field": "rating_int", "gt": 48}
- ]
- }
- ]
+ {"op": "range", "field": "rating_int", "gt": 48},
+ ],
+ },
+ ],
}
self.assertEqual(self._search(filters), [1, 3])
@@ -181,8 +284,8 @@ class TestFilterOpsComplex(unittest.TestCase):
"conds": [
{"op": "range", "field": "price", "lt": 5000},
{"op": "must_not", "field": "category", "conds": ["clothing"]},
- {"op": "range", "field": "rating_int", "gte": 40}
- ]
+ {"op": "range", "field": "rating_int", "gte": 40},
+ ],
}
# Price < 5000: 2, 4, 5, 6, 7, 8, 9, 10
# Not clothing: 1, 2, 3, 4, 5, 6, 7, 8 (excludes 9, 10)
@@ -200,8 +303,8 @@ class TestFilterOpsComplex(unittest.TestCase):
"op": "and",
"conds": [
{"op": "prefix", "field": "tags", "prefix": "f"},
- {"op": "range", "field": "price", "gte": 50, "lte": 100}
- ]
+ {"op": "range", "field": "price", "gte": 50, "lte": 100},
+ ],
}
self.assertEqual(self._search(filters), [7, 8])
@@ -210,21 +313,21 @@ class TestFilterOpsComplex(unittest.TestCase):
# Logic:
# ((Category="electronics" OR Category="home") AND
# (Price < 1000 OR (Tags contains "fiction" AND Rating > 4.5)))
-
+
# A: Category="electronics" -> 1, 2, 3, 4
# B: Category="home" -> 5, 6
# A OR B -> 1, 2, 3, 4, 5, 6
-
+
# C: Price < 1000 -> 6(300), 7(50), 8(60), 9(100), 10(200)
# D: Tags contains "fiction" -> 7, 8
# E: Rating > 4.5 -> 1(48), 3(49), 7(47), 8(46)
# D AND E -> 7, 8
# C OR (D AND E) -> 6, 7, 8, 9, 10 (7,8 already in C, so union is same as C)
-
+
# Intersection (A OR B) AND (C OR (D AND E)):
# {1, 2, 3, 4, 5, 6} INTERSECT {6, 7, 8, 9, 10}
# Result: 6
-
+
filters = {
"op": "and",
"conds": [
@@ -232,8 +335,8 @@ class TestFilterOpsComplex(unittest.TestCase):
"op": "or",
"conds": [
{"op": "must", "field": "category", "conds": ["electronics"]},
- {"op": "must", "field": "category", "conds": ["home"]}
- ]
+ {"op": "must", "field": "category", "conds": ["home"]},
+ ],
},
{
"op": "or",
@@ -243,12 +346,12 @@ class TestFilterOpsComplex(unittest.TestCase):
"op": "and",
"conds": [
{"op": "contains", "field": "tags", "substring": "fiction"},
- {"op": "range", "field": "rating_int", "gt": 45}
- ]
- }
- ]
- }
- ]
+ {"op": "range", "field": "rating_int", "gt": 45},
+ ],
+ },
+ ],
+ },
+ ],
}
self.assertEqual(self._search(filters), [6])
@@ -260,33 +363,25 @@ class TestFilterOpsComplex(unittest.TestCase):
# < 3000: 5, 6, 7, 8, 9, 10
# Result: 3, 5, 6, 7, 8, 9, 10
# (Assuming range_out(gte=3000, lte=8000) means NOT [3000, 8000])
-
- filters = {
- "op": "range_out",
- "field": "price",
- "gte": 3000,
- "lte": 8000
- }
+
+ filters = {"op": "range_out", "field": "price", "gte": 3000, "lte": 8000}
res = self._search(filters)
self.assertEqual(res, [3, 5, 6, 7, 8, 9, 10])
-
+
# range_out combined with Must
# (price < 3000 OR price > 8000) AND Category="electronics"
# Electronics: 1, 2, 3, 4
# Intersection with above: 3
filters_combined = {
"op": "and",
- "conds": [
- filters,
- {"op": "must", "field": "category", "conds": ["electronics"]}
- ]
+ "conds": [filters, {"op": "must", "field": "category", "conds": ["electronics"]}],
}
self.assertEqual(self._search(filters_combined), [3])
def test_multi_layer_logic(self):
"""Test multi-layer logic structure (A OR (B AND (C OR D)))"""
# (Category="books" OR (Category="clothing" AND (Price > 150 OR Rating > 4.2)))
-
+
# A: Category="books" -> 7, 8
# B: Category="clothing" -> 9, 10
# C: Price > 150 -> 1, 2, 3, 4, 5, 6, 10
@@ -294,7 +389,7 @@ class TestFilterOpsComplex(unittest.TestCase):
# C OR D -> 1, 2, 3, 4, 5, 6, 7, 8, 10
# B AND (C OR D) -> {9, 10} INTERSECT {1..8, 10} -> {10}
# A OR (B AND ...) -> {7, 8} UNION {10} -> {7, 8, 10}
-
+
filters = {
"op": "or",
"conds": [
@@ -307,29 +402,29 @@ class TestFilterOpsComplex(unittest.TestCase):
"op": "or",
"conds": [
{"op": "range", "field": "price", "gt": 150},
- {"op": "range", "field": "rating_int", "gt": 42}
- ]
- }
- ]
- }
- ]
+ {"op": "range", "field": "rating_int", "gt": 42},
+ ],
+ },
+ ],
+ },
+ ],
}
self.assertEqual(self._search(filters), [7, 8, 10])
def test_mixed_type_logic(self):
"""Test mixed type field filtering (String Prefix + Int Range + Logic)"""
# (Tags prefix "mobile" AND Price < 5000) OR (Tags prefix "kitchen" AND Price < 500)
-
+
# Part 1: Tags prefix "mobile" -> 1, 2
# Price < 5000 -> 2, 4, 5, 6, 7, 8, 9, 10
# Intersection -> 2
-
+
# Part 2: Tags prefix "kitchen" -> 6
# Price < 500 -> 6, 7, 8, 9, 10
# Intersection -> 6
-
+
# Union -> 2, 6
-
+
filters = {
"op": "or",
"conds": [
@@ -337,17 +432,17 @@ class TestFilterOpsComplex(unittest.TestCase):
"op": "and",
"conds": [
{"op": "prefix", "field": "tags", "prefix": "mobile"},
- {"op": "range", "field": "price", "lt": 5000}
- ]
+ {"op": "range", "field": "price", "lt": 5000},
+ ],
},
{
"op": "and",
"conds": [
{"op": "prefix", "field": "tags", "prefix": "kitchen"},
- {"op": "range", "field": "price", "lt": 500}
- ]
- }
- ]
+ {"op": "range", "field": "price", "lt": 500},
+ ],
+ },
+ ],
}
self.assertEqual(self._search(filters), [2, 6])
@@ -363,8 +458,8 @@ class TestFilterOpsComplex(unittest.TestCase):
"conds": [
{"op": "must", "field": "category", "conds": ["books"]},
{"op": "must", "field": "category", "conds": ["clothing"]},
- {"op": "range", "field": "price", "gt": 10000}
- ]
+ {"op": "range", "field": "price", "gt": 10000},
+ ],
}
self.assertEqual(self._search(filters), [3, 7, 8, 9, 10])
@@ -378,15 +473,15 @@ class TestFilterOpsComplex(unittest.TestCase):
"op": "and",
"conds": [
{"op": "must_not", "field": "category", "conds": ["electronics"]},
- {"op": "range", "field": "price", "gte": 100} # Equivalent to MustNot(Price < 100)
- ]
+ {"op": "range", "field": "price", "gte": 100}, # Equivalent to MustNot(Price < 100)
+ ],
}
self.assertEqual(self._search(filters), [5, 6, 9, 10])
-
class TestFilterOpsLifecycle(unittest.TestCase):
"""Insert/update/delete and restart tests"""
+
def setUp(self):
clean_dir(db_path_lifecycle)
self.path = db_path_lifecycle
@@ -431,17 +526,16 @@ class TestFilterOpsLifecycle(unittest.TestCase):
def _search(self, filters, coll=None):
c = coll if coll else self.collection
res = c.search_by_vector(
- "idx_lifecycle",
- dense_vector=[1.0, 0, 0, 0],
- limit=100,
- filters=filters
+ "idx_lifecycle", dense_vector=[1.0, 0, 0, 0], limit=100, filters=filters
)
return sorted([item.id for item in res.data])
def test_update_impact(self):
"""Test Update impact on Filter"""
# Initial state: status=active -> 1, 2
- self.assertEqual(self._search({"op": "must", "field": "status", "conds": ["active"]}), [1, 2])
+ self.assertEqual(
+ self._search({"op": "must", "field": "status", "conds": ["active"]}), [1, 2]
+ )
# Update id=2 status to inactive
# Update id=3 status to active
@@ -452,7 +546,9 @@ class TestFilterOpsLifecycle(unittest.TestCase):
self.collection.upsert_data(updates)
# Verify after update: status=active -> 1, 3
- self.assertEqual(self._search({"op": "must", "field": "status", "conds": ["active"]}), [1, 3])
+ self.assertEqual(
+ self._search({"op": "must", "field": "status", "conds": ["active"]}), [1, 3]
+ )
# Verify count update: count > 30 -> 3 (35)
self.assertEqual(self._search({"op": "range", "field": "count", "gt": 30}), [3])
@@ -479,7 +575,7 @@ class TestFilterOpsLifecycle(unittest.TestCase):
# Ensure data is written
# Simulate restart: release old object, reload
del self.collection
- self.collection = None # Avoid tearDown accessing deleted attribute
+ self.collection = None # Avoid tearDown accessing deleted attribute
gc.collect()
time.sleep(0.1)
@@ -500,9 +596,9 @@ class TestFilterOpsLifecycle(unittest.TestCase):
# tearDown will handle drop
-
class TestFilterOpsPath(unittest.TestCase):
"""Path type Filter tests"""
+
def setUp(self):
clean_dir("./db_test_filters_path/")
self.path = "./db_test_filters_path/"
@@ -547,28 +643,35 @@ class TestFilterOpsPath(unittest.TestCase):
def _search(self, filters):
res = self.collection.search_by_vector(
- "idx_path",
- dense_vector=[1.0, 0, 0, 0],
- limit=100,
- filters=filters
+ "idx_path", dense_vector=[1.0, 0, 0, 0], limit=100, filters=filters
)
return sorted([item.id for item in res.data])
def test_path_must(self):
"""Test Must matching path prefix"""
# Must /a -> /a/b/c, /a/b/d, /a/e
- self.assertEqual(self._search({"op": "must", "field": "file_path", "conds": ["/a"]}), [1, 2, 3])
+ self.assertEqual(
+ self._search({"op": "must", "field": "file_path", "conds": ["/a"]}), [1, 2, 3]
+ )
# Must /a/b -> /a/b/c, /a/b/d
- self.assertEqual(self._search({"op": "must", "field": "file_path", "conds": ["/a/b"]}), [1, 2])
+ self.assertEqual(
+ self._search({"op": "must", "field": "file_path", "conds": ["/a/b"]}), [1, 2]
+ )
# Must /f -> /f/g, /f/h/i
- self.assertEqual(self._search({"op": "must", "field": "file_path", "conds": ["/f"]}), [4, 5])
+ self.assertEqual(
+ self._search({"op": "must", "field": "file_path", "conds": ["/f"]}), [4, 5]
+ )
def test_path_must_not(self):
"""Test MustNot excluding path prefix"""
# MustNot /a/b -> exclude 1, 2 -> remaining 3, 4, 5
- self.assertEqual(self._search({"op": "must_not", "field": "file_path", "conds": ["/a/b"]}), [3, 4, 5])
+ self.assertEqual(
+ self._search({"op": "must_not", "field": "file_path", "conds": ["/a/b"]}), [3, 4, 5]
+ )
# MustNot /a -> exclude 1, 2, 3 -> remaining 4, 5
- self.assertEqual(self._search({"op": "must_not", "field": "file_path", "conds": ["/a"]}), [4, 5])
+ self.assertEqual(
+ self._search({"op": "must_not", "field": "file_path", "conds": ["/a"]}), [4, 5]
+ )
def test_path_depth(self):
"""Test path type depth parameter"""
@@ -607,28 +710,28 @@ class TestFilterOpsPath(unittest.TestCase):
# Must /a, depth=1
# /a/e -> Yes
# /a/b/c -> No (depth 2)
-
+
self.assertEqual(
- self._search({"op": "must", "field": "file_path", "conds": ["/a"], "para": "-d=1"}),
- [3]
+ self._search({"op": "must", "field": "file_path", "conds": ["/a"], "para": "-d=1"}), [3]
)
# Must /a, depth=2 (default recursive all? Or specified depth)
# /a/e (depth 1) -> Yes
# /a/b/c (depth 2) -> Yes
self.assertEqual(
- self._search({"op": "must", "field": "file_path", "conds": ["/a"], "para": "-d=2"}),
- [1, 2, 3]
+ self._search({"op": "must", "field": "file_path", "conds": ["/a"], "para": "-d=2"}),
+ [1, 2, 3],
)
class TestFilterOpsScale(unittest.TestCase):
"""Scale tests"""
+
def setUp(self):
clean_dir(db_path_scale)
self.path = db_path_scale
self.collection = self._create_collection()
- self.data_count = 500000 # 50w
+ self.data_count = 500000 # 50w
self._insert_large_data()
self._create_index()
@@ -657,35 +760,37 @@ class TestFilterOpsScale(unittest.TestCase):
# SubGroup: X (even id), Y (odd id)
# Score: id % 100
# Tag: prefix_{id%1000}
-
+
batch_size = 10000
data = []
print(f"Inserting {self.data_count} items...")
start_time = time.time()
-
+
for i in range(self.data_count):
group = "A" if i < self.data_count / 2 else "B"
sub_group = "X" if i % 2 == 0 else "Y"
score = i % 100
tag = f"tag_{i % 1000}"
-
- data.append({
- "id": i,
- "embedding": [random.random() for _ in range(4)],
- "group": group,
- "sub_group": sub_group,
- "score": score,
- "tag": tag
- })
-
+
+ data.append(
+ {
+ "id": i,
+ "embedding": [random.random() for _ in range(4)],
+ "group": group,
+ "sub_group": sub_group,
+ "score": score,
+ "tag": tag,
+ }
+ )
+
if len(data) >= batch_size:
self.collection.upsert_data(data)
data = []
# print(f"Inserted {i+1} items")
-
+
if data:
self.collection.upsert_data(data)
-
+
print(f"Insert finished in {time.time() - start_time:.2f}s")
def _create_index(self):
@@ -702,10 +807,7 @@ class TestFilterOpsScale(unittest.TestCase):
def _search(self, filters, limit=10000):
# For large scale test, we just check count mainly
res = self.collection.search_by_vector(
- "idx_scale",
- dense_vector=[0.0, 0.0, 0.0, 0.0],
- limit=limit,
- filters=filters
+ "idx_scale", dense_vector=[0.0, 0.0, 0.0, 0.0], limit=limit, filters=filters
)
return [item.id for item in res.data]
@@ -721,23 +823,23 @@ class TestFilterOpsScale(unittest.TestCase):
"op": "and",
"conds": [
{"op": "must", "field": "group", "conds": ["A"]},
- {"op": "range", "field": "score", "gt": 90}
- ]
+ {"op": "range", "field": "score", "gt": 90},
+ ],
}
# limit needs to be large enough to catch all or we verify count returned is limit
# If limit < expected, we get limit. If limit > expected, we get expected.
# Let's set a large limit for counting test, but verify performance
# 22500 results is large. Let's just sample top 100 but rely on a smaller range for exact count verification
# or verify standard limit behavior.
-
+
# Search with smaller result set for verification
# Group A AND Score = 99 (1 per 100) -> 2500 expected
filters = {
"op": "and",
"conds": [
{"op": "must", "field": "group", "conds": ["A"]},
- {"op": "must", "field": "score", "conds": [99]}
- ]
+ {"op": "must", "field": "score", "conds": [99]},
+ ],
}
ids = self._search(filters, limit=5000)
print(f"Filter 1 time: {time.time() - start_time:.4f}s")
@@ -752,22 +854,22 @@ class TestFilterOpsScale(unittest.TestCase):
# Group B (250000-499999) AND Score < 5 (0,1,2,3,4 -> 5 per 100) -> 2500 * 5 = 12500 items
# Total = 137500 items. Too many to fetch all.
# Let's add more conditions to reduce result set.
-
- # (Group A AND SubGroup X AND Tag="tag_0")
+
+ # (Group A AND SubGroup X AND Tag="tag_0")
# Tag="tag_0" -> id % 1000 == 0.
# Group A (0-249999): 250 items with tag_0.
# SubGroup X (Even): tag_0 implies id%1000=0 which is even. So all 250 items match X.
# Result: 250 items.
-
+
# OR
-
+
# (Group B AND Score=0 AND SubGroup Y)
# Group B (250000-499999)
# Score=0 -> id % 100 == 0.
# SubGroup Y (Odd) -> id is Odd.
# id % 100 == 0 implies id is Even. So (Even AND Odd) -> Empty.
# Result: 0 items.
-
+
# Total expected: 250 items.
print("Testing Filter 2: Complex Nested Logic")
filters = {
@@ -778,18 +880,18 @@ class TestFilterOpsScale(unittest.TestCase):
"conds": [
{"op": "must", "field": "group", "conds": ["A"]},
{"op": "must", "field": "sub_group", "conds": ["X"]},
- {"op": "must", "field": "tag", "conds": ["tag_0"]}
- ]
+ {"op": "must", "field": "tag", "conds": ["tag_0"]},
+ ],
},
{
"op": "and",
"conds": [
{"op": "must", "field": "group", "conds": ["B"]},
{"op": "must", "field": "score", "conds": [0]},
- {"op": "must", "field": "sub_group", "conds": ["Y"]}
- ]
- }
- ]
+ {"op": "must", "field": "sub_group", "conds": ["Y"]},
+ ],
+ },
+ ],
}
start_time = time.time()
ids = self._search(filters, limit=1000)
@@ -812,5 +914,6 @@ class TestFilterOpsScale(unittest.TestCase):
for i in ids:
self.assertEqual(i % 1000, 999)
-if __name__ == '__main__':
+
+if __name__ == "__main__":
unittest.main()
diff --git a/tests/vectordb/test_project_group.py b/tests/vectordb/test_project_group.py
index 53f7ee4a3..093956e65 100644
--- a/tests/vectordb/test_project_group.py
+++ b/tests/vectordb/test_project_group.py
@@ -3,10 +3,12 @@
import os
import shutil
import unittest
-from openviking.storage.vectordb.project.project_group import ProjectGroup, get_or_create_project_group
+
+from openviking.storage.vectordb.project.project_group import get_or_create_project_group
TEST_PROJECT_ROOT = "./test_project_root"
+
class TestProjectGroup(unittest.TestCase):
def setUp(self):
if os.path.exists(TEST_PROJECT_ROOT):
@@ -20,40 +22,40 @@ class TestProjectGroup(unittest.TestCase):
# Path empty -> Volatile
group = get_or_create_project_group("")
self.assertTrue(group.has_project("default"))
-
+
# Create new
p1 = group.create_project("p1")
self.assertIsNotNone(p1)
self.assertTrue(group.has_project("p1"))
-
+
# List
names = group.list_projects()
self.assertIn("default", names)
self.assertIn("p1", names)
-
+
# Close
group.close()
def test_persistent_group_lifecycle(self):
# 1. Create and populate
group = get_or_create_project_group(TEST_PROJECT_ROOT)
-
+
# Default should be created automatically
self.assertTrue(group.has_project("default"))
self.assertTrue(os.path.exists(os.path.join(TEST_PROJECT_ROOT, "default")))
-
+
# Create persistent project
group.create_project("analytics")
self.assertTrue(os.path.exists(os.path.join(TEST_PROJECT_ROOT, "analytics")))
-
+
# Close to flush/release
group.close()
-
+
# 2. Reload from disk
group2 = get_or_create_project_group(TEST_PROJECT_ROOT)
self.assertTrue(group2.has_project("default"))
self.assertTrue(group2.has_project("analytics"))
-
+
# 3. Delete project
group2.delete_project("analytics")
self.assertFalse(group2.has_project("analytics"))
@@ -66,7 +68,7 @@ class TestProjectGroup(unittest.TestCase):
# project.close()
# It does not seem to remove the project directory itself in `project_group.py`.
# However, for robustness, we at least ensure it's gone from memory.
-
+
group2.close()
def test_duplicate_create_error(self):
@@ -76,5 +78,6 @@ class TestProjectGroup(unittest.TestCase):
group.create_project("dup_test")
group.close()
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/vectordb/test_pydantic_validation.py b/tests/vectordb/test_pydantic_validation.py
index 1352295ea..66d9f41e1 100644
--- a/tests/vectordb/test_pydantic_validation.py
+++ b/tests/vectordb/test_pydantic_validation.py
@@ -1,13 +1,14 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
-import sys
import os
+import sys
import unittest
+from openviking.storage.vectordb.utils import validation
+
sys.path.append(os.getcwd())
-from openviking.storage.vectordb.utils import validation
class TestPydanticValidation(unittest.TestCase):
def test_valid_collection_meta(self):
@@ -16,8 +17,8 @@ class TestPydanticValidation(unittest.TestCase):
"Fields": [
{"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},
{"FieldName": "vec", "FieldType": "vector", "Dim": 128},
- {"FieldName": "desc", "FieldType": "string"}
- ]
+ {"FieldName": "desc", "FieldType": "string"},
+ ],
}
self.assertTrue(validation.is_valid_collection_meta_data(meta))
@@ -25,41 +26,37 @@ class TestPydanticValidation(unittest.TestCase):
# Missing required field
meta = {"Fields": []}
self.assertFalse(validation.is_valid_collection_meta_data(meta))
-
+
# Invalid dim
meta = {
"CollectionName": "test",
- "Fields": [{"FieldName": "vec", "FieldType": "vector", "Dim": 129}] # Not multiple of 4
+ "Fields": [
+ {"FieldName": "vec", "FieldType": "vector", "Dim": 129}
+ ], # Not multiple of 4
}
self.assertFalse(validation.is_valid_collection_meta_data(meta))
def test_valid_index_meta(self):
meta = {
"IndexName": "test_index",
- "VectorIndex": {
- "IndexType": "flat",
- "Distance": "L2",
- "Quant": "float"
- }
+ "VectorIndex": {"IndexType": "flat", "Distance": "L2", "Quant": "float"},
}
fields_meta = {}
self.assertTrue(validation.is_valid_index_meta_data(meta, fields_meta))
def test_fix_collection_meta(self):
- meta = {
- "CollectionName": "test",
- "Fields": [
- {"FieldName": "text", "FieldType": "string"}
- ]
- }
+ meta = {"CollectionName": "test", "Fields": [{"FieldName": "text", "FieldType": "string"}]}
fixed = validation.fix_collection_meta(meta)
-
+
# Check if AUTO_ID was added
- has_auto_id = any(f["FieldName"] == "AUTO_ID" and f["IsPrimaryKey"] for f in fixed["Fields"])
+ has_auto_id = any(
+ f["FieldName"] == "AUTO_ID" and f["IsPrimaryKey"] for f in fixed["Fields"]
+ )
self.assertTrue(has_auto_id)
-
+
# Check _FieldID assignment
self.assertIn("_FieldID", fixed["Fields"][0])
-if __name__ == '__main__':
+
+if __name__ == "__main__":
unittest.main()
diff --git a/tests/vectordb/test_recall.py b/tests/vectordb/test_recall.py
index 9501aa86a..cdc5c39e7 100644
--- a/tests/vectordb/test_recall.py
+++ b/tests/vectordb/test_recall.py
@@ -1,21 +1,24 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
-import unittest
-import shutil
import random
-import math
-from typing import List, Tuple
+import shutil
+import unittest
+from typing import List
+
from openviking.storage.vectordb.collection.local_collection import get_or_create_local_collection
# Test data path
TEST_DB_PATH = "./test_recall_collection/"
+
def calculate_l2_distance(v1: List[float], v2: List[float]) -> float:
return sum((a - b) ** 2 for a, b in zip(v1, v2))
+
def calculate_ip_distance(v1: List[float], v2: List[float]) -> float:
return sum(a * b for a, b in zip(v1, v2))
+
class TestRecall(unittest.TestCase):
"""Test vector recall quality"""
@@ -41,7 +44,7 @@ class TestRecall(unittest.TestCase):
def test_exact_match_recall(self):
"""Test if the exact vector is recalled at rank 1"""
print("\n=== Test: Exact Match Recall ===")
-
+
dim = 64
meta_data = {
"CollectionName": "test_exact_match",
@@ -73,26 +76,28 @@ class TestRecall(unittest.TestCase):
{
"IndexName": "idx_flat",
"VectorIndex": {"IndexType": "flat", "Distance": "l2"},
- }
+ },
)
# Query with an existing vector
target_idx = 500
query_vec = vectors[target_idx]
-
+
result = collection.search_by_vector("idx_flat", dense_vector=query_vec, limit=10)
-
+
self.assertTrue(len(result.data) > 0)
# The first result should be the vector itself (id=500)
- # Note: Depending on floating point precision, distance might not be exactly 0.0,
+ # Note: Depending on floating point precision, distance might not be exactly 0.0,
# but it should be the closest.
- self.assertEqual(result.data[0].id, target_idx, "The top result should be the query vector itself")
+ self.assertEqual(
+ result.data[0].id, target_idx, "The top result should be the query vector itself"
+ )
print("βœ“ Exact match verified")
def test_l2_recall_topk(self):
"""Test Top-K recall for L2 distance"""
print("\n=== Test: Top-K Recall (L2) ===")
-
+
dim = 32
total_records = 500
meta_data = {
@@ -115,53 +120,55 @@ class TestRecall(unittest.TestCase):
vec = [random.uniform(0, 1) for _ in range(dim)]
vectors.append(vec)
data.append({"id": i, "vector": vec})
-
+
collection.upsert_data(data)
-
+
collection.create_index(
"idx_l2",
{
"IndexName": "idx_l2",
"VectorIndex": {"IndexType": "flat", "Distance": "l2"},
- }
+ },
)
# Generate a query vector
query_vec = [random.uniform(0, 1) for _ in range(dim)]
-
+
# Calculate Ground Truth
# (distance, id)
distances = []
for i, vec in enumerate(vectors):
dist = calculate_l2_distance(query_vec, vec)
distances.append((dist, i))
-
+
# Sort by distance ascending (L2)
distances.sort(key=lambda x: x[0])
ground_truth_ids = [x[1] for x in distances[:10]]
-
+
# Search
result = collection.search_by_vector("idx_l2", dense_vector=query_vec, limit=10)
result_ids = [item.id for item in result.data]
-
+
print(f"Ground Truth IDs: {ground_truth_ids}")
print(f"Search Result IDs: {result_ids}")
-
+
# Calculate Recall@10
intersection = set(ground_truth_ids) & set(result_ids)
recall = len(intersection) / 10.0
print(f"Recall@10: {recall}")
-
+
self.assertEqual(recall, 1.0, "Recall@10 for Flat index should be 1.0")
-
+
# Verify order matches
- self.assertEqual(result_ids, ground_truth_ids, "Result order should match ground truth for Flat index")
+ self.assertEqual(
+ result_ids, ground_truth_ids, "Result order should match ground truth for Flat index"
+ )
print("βœ“ L2 Recall verified")
def test_ip_recall_topk(self):
"""Test Top-K recall for Inner Product (IP) distance"""
print("\n=== Test: Top-K Recall (IP) ===")
-
+
dim = 32
total_records = 500
meta_data = {
@@ -181,51 +188,54 @@ class TestRecall(unittest.TestCase):
vectors = []
data = []
for i in range(total_records):
- # Normalize vectors for IP to behave like Cosine Similarity if needed,
- # but IP itself is just dot product.
+ # Normalize vectors for IP to behave like Cosine Similarity if needed,
+ # but IP itself is just dot product.
vec = [random.uniform(-1, 1) for _ in range(dim)]
vectors.append(vec)
data.append({"id": i, "vector": vec})
-
+
collection.upsert_data(data)
-
+
collection.create_index(
"idx_ip",
{
"IndexName": "idx_ip",
"VectorIndex": {"IndexType": "flat", "Distance": "ip"},
- }
+ },
)
# Generate a query vector
query_vec = [random.uniform(-1, 1) for _ in range(dim)]
-
+
# Calculate Ground Truth
# (score, id)
scores = []
for i, vec in enumerate(vectors):
score = calculate_ip_distance(query_vec, vec)
scores.append((score, i))
-
+
# Sort by score descending (IP)
scores.sort(key=lambda x: x[0], reverse=True)
ground_truth_ids = [x[1] for x in scores[:10]]
-
+
# Search
result = collection.search_by_vector("idx_ip", dense_vector=query_vec, limit=10)
result_ids = [item.id for item in result.data]
-
+
print(f"Ground Truth IDs: {ground_truth_ids}")
print(f"Search Result IDs: {result_ids}")
-
+
# Calculate Recall@10
intersection = set(ground_truth_ids) & set(result_ids)
recall = len(intersection) / 10.0
print(f"Recall@10: {recall}")
-
+
self.assertEqual(recall, 1.0, "Recall@10 for Flat index should be 1.0")
- self.assertEqual(result_ids, ground_truth_ids, "Result order should match ground truth for Flat index")
+ self.assertEqual(
+ result_ids, ground_truth_ids, "Result order should match ground truth for Flat index"
+ )
print("βœ“ IP Recall verified")
+
if __name__ == "__main__":
unittest.main()
diff --git a/uv.lock b/uv.lock
index 708fa5c89..9729818a3 100644
--- a/uv.lock
+++ b/uv.lock
@@ -113,103 +113,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" },
]
-[[package]]
-name = "black"
-version = "25.11.0"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
- "python_full_version < '3.10'",
-]
-dependencies = [
- { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
- { name = "mypy-extensions", marker = "python_full_version < '3.10'" },
- { name = "packaging", marker = "python_full_version < '3.10'" },
- { name = "pathspec", marker = "python_full_version < '3.10'" },
- { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
- { name = "pytokens", marker = "python_full_version < '3.10'" },
- { name = "tomli", marker = "python_full_version < '3.10'" },
- { name = "typing-extensions", marker = "python_full_version < '3.10'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/8c/ad/33adf4708633d047950ff2dfdea2e215d84ac50ef95aff14a614e4b6e9b2/black-25.11.0.tar.gz", hash = "sha256:9a323ac32f5dc75ce7470501b887250be5005a01602e931a15e45593f70f6e08", size = 655669, upload-time = "2025-11-10T01:53:50.558Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/b3/d2/6caccbc96f9311e8ec3378c296d4f4809429c43a6cd2394e3c390e86816d/black-25.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ec311e22458eec32a807f029b2646f661e6859c3f61bc6d9ffb67958779f392e", size = 1743501, upload-time = "2025-11-10T01:59:06.202Z" },
- { url = "https://files.pythonhosted.org/packages/69/35/b986d57828b3f3dccbf922e2864223197ba32e74c5004264b1c62bc9f04d/black-25.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1032639c90208c15711334d681de2e24821af0575573db2810b0763bcd62e0f0", size = 1597308, upload-time = "2025-11-10T01:57:58.633Z" },
- { url = "https://files.pythonhosted.org/packages/39/8e/8b58ef4b37073f52b64a7b2dd8c9a96c84f45d6f47d878d0aa557e9a2d35/black-25.11.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0c0f7c461df55cf32929b002335883946a4893d759f2df343389c4396f3b6b37", size = 1656194, upload-time = "2025-11-10T01:57:10.909Z" },
- { url = "https://files.pythonhosted.org/packages/8d/30/9c2267a7955ecc545306534ab88923769a979ac20a27cf618d370091e5dd/black-25.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:f9786c24d8e9bd5f20dc7a7f0cdd742644656987f6ea6947629306f937726c03", size = 1347996, upload-time = "2025-11-10T01:57:22.391Z" },
- { url = "https://files.pythonhosted.org/packages/c4/62/d304786b75ab0c530b833a89ce7d997924579fb7484ecd9266394903e394/black-25.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:895571922a35434a9d8ca67ef926da6bc9ad464522a5fe0db99b394ef1c0675a", size = 1727891, upload-time = "2025-11-10T02:01:40.507Z" },
- { url = "https://files.pythonhosted.org/packages/82/5d/ffe8a006aa522c9e3f430e7b93568a7b2163f4b3f16e8feb6d8c3552761a/black-25.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cb4f4b65d717062191bdec8e4a442539a8ea065e6af1c4f4d36f0cdb5f71e170", size = 1581875, upload-time = "2025-11-10T01:57:51.192Z" },
- { url = "https://files.pythonhosted.org/packages/cb/c8/7c8bda3108d0bb57387ac41b4abb5c08782b26da9f9c4421ef6694dac01a/black-25.11.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d81a44cbc7e4f73a9d6ae449ec2317ad81512d1e7dce7d57f6333fd6259737bc", size = 1642716, upload-time = "2025-11-10T01:56:51.589Z" },
- { url = "https://files.pythonhosted.org/packages/34/b9/f17dea34eecb7cc2609a89627d480fb6caea7b86190708eaa7eb15ed25e7/black-25.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:7eebd4744dfe92ef1ee349dc532defbf012a88b087bb7ddd688ff59a447b080e", size = 1352904, upload-time = "2025-11-10T01:59:26.252Z" },
- { url = "https://files.pythonhosted.org/packages/7f/12/5c35e600b515f35ffd737da7febdb2ab66bb8c24d88560d5e3ef3d28c3fd/black-25.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:80e7486ad3535636657aa180ad32a7d67d7c273a80e12f1b4bfa0823d54e8fac", size = 1772831, upload-time = "2025-11-10T02:03:47Z" },
- { url = "https://files.pythonhosted.org/packages/1a/75/b3896bec5a2bb9ed2f989a970ea40e7062f8936f95425879bbe162746fe5/black-25.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6cced12b747c4c76bc09b4db057c319d8545307266f41aaee665540bc0e04e96", size = 1608520, upload-time = "2025-11-10T01:58:46.895Z" },
- { url = "https://files.pythonhosted.org/packages/f3/b5/2bfc18330eddbcfb5aab8d2d720663cd410f51b2ed01375f5be3751595b0/black-25.11.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6cb2d54a39e0ef021d6c5eef442e10fd71fcb491be6413d083a320ee768329dd", size = 1682719, upload-time = "2025-11-10T01:56:55.24Z" },
- { url = "https://files.pythonhosted.org/packages/96/fb/f7dc2793a22cdf74a72114b5ed77fe3349a2e09ef34565857a2f917abdf2/black-25.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:ae263af2f496940438e5be1a0c1020e13b09154f3af4df0835ea7f9fe7bfa409", size = 1362684, upload-time = "2025-11-10T01:57:07.639Z" },
- { url = "https://files.pythonhosted.org/packages/ad/47/3378d6a2ddefe18553d1115e36aea98f4a90de53b6a3017ed861ba1bd3bc/black-25.11.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0a1d40348b6621cc20d3d7530a5b8d67e9714906dfd7346338249ad9c6cedf2b", size = 1772446, upload-time = "2025-11-10T02:02:16.181Z" },
- { url = "https://files.pythonhosted.org/packages/ba/4b/0f00bfb3d1f7e05e25bfc7c363f54dc523bb6ba502f98f4ad3acf01ab2e4/black-25.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:51c65d7d60bb25429ea2bf0731c32b2a2442eb4bd3b2afcb47830f0b13e58bfd", size = 1607983, upload-time = "2025-11-10T02:02:52.502Z" },
- { url = "https://files.pythonhosted.org/packages/99/fe/49b0768f8c9ae57eb74cc10a1f87b4c70453551d8ad498959721cc345cb7/black-25.11.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:936c4dd07669269f40b497440159a221ee435e3fddcf668e0c05244a9be71993", size = 1682481, upload-time = "2025-11-10T01:57:12.35Z" },
- { url = "https://files.pythonhosted.org/packages/55/17/7e10ff1267bfa950cc16f0a411d457cdff79678fbb77a6c73b73a5317904/black-25.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:f42c0ea7f59994490f4dccd64e6b2dd49ac57c7c84f38b8faab50f8759db245c", size = 1363869, upload-time = "2025-11-10T01:58:24.608Z" },
- { url = "https://files.pythonhosted.org/packages/67/c0/cc865ce594d09e4cd4dfca5e11994ebb51604328489f3ca3ae7bb38a7db5/black-25.11.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:35690a383f22dd3e468c85dc4b915217f87667ad9cce781d7b42678ce63c4170", size = 1771358, upload-time = "2025-11-10T02:03:33.331Z" },
- { url = "https://files.pythonhosted.org/packages/37/77/4297114d9e2fd2fc8ab0ab87192643cd49409eb059e2940391e7d2340e57/black-25.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dae49ef7369c6caa1a1833fd5efb7c3024bb7e4499bf64833f65ad27791b1545", size = 1612902, upload-time = "2025-11-10T01:59:33.382Z" },
- { url = "https://files.pythonhosted.org/packages/de/63/d45ef97ada84111e330b2b2d45e1dd163e90bd116f00ac55927fb6bf8adb/black-25.11.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bd4a22a0b37401c8e492e994bce79e614f91b14d9ea911f44f36e262195fdda", size = 1680571, upload-time = "2025-11-10T01:57:04.239Z" },
- { url = "https://files.pythonhosted.org/packages/ff/4b/5604710d61cdff613584028b4cb4607e56e148801ed9b38ee7970799dab6/black-25.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:aa211411e94fdf86519996b7f5f05e71ba34835d8f0c0f03c00a26271da02664", size = 1382599, upload-time = "2025-11-10T01:57:57.427Z" },
- { url = "https://files.pythonhosted.org/packages/d5/9a/5b2c0e3215fe748fcf515c2dd34658973a1210bf610e24de5ba887e4f1c8/black-25.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a3bb5ce32daa9ff0605d73b6f19da0b0e6c1f8f2d75594db539fdfed722f2b06", size = 1743063, upload-time = "2025-11-10T02:02:43.175Z" },
- { url = "https://files.pythonhosted.org/packages/a1/20/245164c6efc27333409c62ba54dcbfbe866c6d1957c9a6c0647786e950da/black-25.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9815ccee1e55717fe9a4b924cae1646ef7f54e0f990da39a34fc7b264fcf80a2", size = 1596867, upload-time = "2025-11-10T02:00:17.157Z" },
- { url = "https://files.pythonhosted.org/packages/ca/6f/1a3859a7da205f3d50cf3a8bec6bdc551a91c33ae77a045bb24c1f46ab54/black-25.11.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92285c37b93a1698dcbc34581867b480f1ba3a7b92acf1fe0467b04d7a4da0dc", size = 1655678, upload-time = "2025-11-10T01:57:09.028Z" },
- { url = "https://files.pythonhosted.org/packages/56/1a/6dec1aeb7be90753d4fcc273e69bc18bfd34b353223ed191da33f7519410/black-25.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:43945853a31099c7c0ff8dface53b4de56c41294fa6783c0441a8b1d9bf668bc", size = 1347452, upload-time = "2025-11-10T01:57:01.871Z" },
- { url = "https://files.pythonhosted.org/packages/00/5d/aed32636ed30a6e7f9efd6ad14e2a0b0d687ae7c8c7ec4e4a557174b895c/black-25.11.0-py3-none-any.whl", hash = "sha256:e3f562da087791e96cefcd9dda058380a442ab322a02e222add53736451f604b", size = 204918, upload-time = "2025-11-10T01:53:48.917Z" },
-]
-
-[[package]]
-name = "black"
-version = "25.12.0"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
- "python_full_version >= '3.14'",
- "python_full_version == '3.13.*'",
- "python_full_version == '3.12.*'",
- "python_full_version == '3.11.*'",
- "python_full_version == '3.10.*'",
-]
-dependencies = [
- { name = "click", version = "8.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
- { name = "mypy-extensions", marker = "python_full_version >= '3.10'" },
- { name = "packaging", marker = "python_full_version >= '3.10'" },
- { name = "pathspec", marker = "python_full_version >= '3.10'" },
- { name = "platformdirs", version = "4.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
- { name = "pytokens", marker = "python_full_version >= '3.10'" },
- { name = "tomli", marker = "python_full_version == '3.10.*'" },
- { name = "typing-extensions", marker = "python_full_version == '3.10.*'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/c4/d9/07b458a3f1c525ac392b5edc6b191ff140b596f9d77092429417a54e249d/black-25.12.0.tar.gz", hash = "sha256:8d3dd9cea14bff7ddc0eb243c811cdb1a011ebb4800a5f0335a01a68654796a7", size = 659264, upload-time = "2025-12-08T01:40:52.501Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/37/d5/8d3145999d380e5d09bb00b0f7024bf0a8ccb5c07b5648e9295f02ec1d98/black-25.12.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f85ba1ad15d446756b4ab5f3044731bf68b777f8f9ac9cdabd2425b97cd9c4e8", size = 1895720, upload-time = "2025-12-08T01:46:58.197Z" },
- { url = "https://files.pythonhosted.org/packages/06/97/7acc85c4add41098f4f076b21e3e4e383ad6ed0a3da26b2c89627241fc11/black-25.12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:546eecfe9a3a6b46f9d69d8a642585a6eaf348bcbbc4d87a19635570e02d9f4a", size = 1727193, upload-time = "2025-12-08T01:52:26.674Z" },
- { url = "https://files.pythonhosted.org/packages/24/f0/fdf0eb8ba907ddeb62255227d29d349e8256ef03558fbcadfbc26ecfe3b2/black-25.12.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:17dcc893da8d73d8f74a596f64b7c98ef5239c2cd2b053c0f25912c4494bf9ea", size = 1774506, upload-time = "2025-12-08T01:46:25.721Z" },
- { url = "https://files.pythonhosted.org/packages/e4/f5/9203a78efe00d13336786b133c6180a9303d46908a9aa72d1104ca214222/black-25.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:09524b0e6af8ba7a3ffabdfc7a9922fb9adef60fed008c7cd2fc01f3048e6e6f", size = 1416085, upload-time = "2025-12-08T01:46:06.073Z" },
- { url = "https://files.pythonhosted.org/packages/ba/cc/7a6090e6b081c3316282c05c546e76affdce7bf7a3b7d2c3a2a69438bd01/black-25.12.0-cp310-cp310-win_arm64.whl", hash = "sha256:b162653ed89eb942758efeb29d5e333ca5bb90e5130216f8369857db5955a7da", size = 1226038, upload-time = "2025-12-08T01:45:29.388Z" },
- { url = "https://files.pythonhosted.org/packages/60/ad/7ac0d0e1e0612788dbc48e62aef8a8e8feffac7eb3d787db4e43b8462fa8/black-25.12.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d0cfa263e85caea2cff57d8f917f9f51adae8e20b610e2b23de35b5b11ce691a", size = 1877003, upload-time = "2025-12-08T01:43:29.967Z" },
- { url = "https://files.pythonhosted.org/packages/e8/dd/a237e9f565f3617a88b49284b59cbca2a4f56ebe68676c1aad0ce36a54a7/black-25.12.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1a2f578ae20c19c50a382286ba78bfbeafdf788579b053d8e4980afb079ab9be", size = 1712639, upload-time = "2025-12-08T01:52:46.756Z" },
- { url = "https://files.pythonhosted.org/packages/12/80/e187079df1ea4c12a0c63282ddd8b81d5107db6d642f7d7b75a6bcd6fc21/black-25.12.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d3e1b65634b0e471d07ff86ec338819e2ef860689859ef4501ab7ac290431f9b", size = 1758143, upload-time = "2025-12-08T01:45:29.137Z" },
- { url = "https://files.pythonhosted.org/packages/93/b5/3096ccee4f29dc2c3aac57274326c4d2d929a77e629f695f544e159bfae4/black-25.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:a3fa71e3b8dd9f7c6ac4d818345237dfb4175ed3bf37cd5a581dbc4c034f1ec5", size = 1420698, upload-time = "2025-12-08T01:45:53.379Z" },
- { url = "https://files.pythonhosted.org/packages/7e/39/f81c0ffbc25ffbe61c7d0385bf277e62ffc3e52f5ee668d7369d9854fadf/black-25.12.0-cp311-cp311-win_arm64.whl", hash = "sha256:51e267458f7e650afed8445dc7edb3187143003d52a1b710c7321aef22aa9655", size = 1229317, upload-time = "2025-12-08T01:46:35.606Z" },
- { url = "https://files.pythonhosted.org/packages/d1/bd/26083f805115db17fda9877b3c7321d08c647df39d0df4c4ca8f8450593e/black-25.12.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:31f96b7c98c1ddaeb07dc0f56c652e25bdedaac76d5b68a059d998b57c55594a", size = 1924178, upload-time = "2025-12-08T01:49:51.048Z" },
- { url = "https://files.pythonhosted.org/packages/89/6b/ea00d6651561e2bdd9231c4177f4f2ae19cc13a0b0574f47602a7519b6ca/black-25.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:05dd459a19e218078a1f98178c13f861fe6a9a5f88fc969ca4d9b49eb1809783", size = 1742643, upload-time = "2025-12-08T01:49:59.09Z" },
- { url = "https://files.pythonhosted.org/packages/6d/f3/360fa4182e36e9875fabcf3a9717db9d27a8d11870f21cff97725c54f35b/black-25.12.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c1f68c5eff61f226934be6b5b80296cf6939e5d2f0c2f7d543ea08b204bfaf59", size = 1800158, upload-time = "2025-12-08T01:44:27.301Z" },
- { url = "https://files.pythonhosted.org/packages/f8/08/2c64830cb6616278067e040acca21d4f79727b23077633953081c9445d61/black-25.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:274f940c147ddab4442d316b27f9e332ca586d39c85ecf59ebdea82cc9ee8892", size = 1426197, upload-time = "2025-12-08T01:45:51.198Z" },
- { url = "https://files.pythonhosted.org/packages/d4/60/a93f55fd9b9816b7432cf6842f0e3000fdd5b7869492a04b9011a133ee37/black-25.12.0-cp312-cp312-win_arm64.whl", hash = "sha256:169506ba91ef21e2e0591563deda7f00030cb466e747c4b09cb0a9dae5db2f43", size = 1237266, upload-time = "2025-12-08T01:45:10.556Z" },
- { url = "https://files.pythonhosted.org/packages/c8/52/c551e36bc95495d2aa1a37d50566267aa47608c81a53f91daa809e03293f/black-25.12.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a05ddeb656534c3e27a05a29196c962877c83fa5503db89e68857d1161ad08a5", size = 1923809, upload-time = "2025-12-08T01:46:55.126Z" },
- { url = "https://files.pythonhosted.org/packages/a0/f7/aac9b014140ee56d247e707af8db0aae2e9efc28d4a8aba92d0abd7ae9d1/black-25.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9ec77439ef3e34896995503865a85732c94396edcc739f302c5673a2315e1e7f", size = 1742384, upload-time = "2025-12-08T01:49:37.022Z" },
- { url = "https://files.pythonhosted.org/packages/74/98/38aaa018b2ab06a863974c12b14a6266badc192b20603a81b738c47e902e/black-25.12.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e509c858adf63aa61d908061b52e580c40eae0dfa72415fa47ac01b12e29baf", size = 1798761, upload-time = "2025-12-08T01:46:05.386Z" },
- { url = "https://files.pythonhosted.org/packages/16/3a/a8ac542125f61574a3f015b521ca83b47321ed19bb63fe6d7560f348bfe1/black-25.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:252678f07f5bac4ff0d0e9b261fbb029fa530cfa206d0a636a34ab445ef8ca9d", size = 1429180, upload-time = "2025-12-08T01:45:34.903Z" },
- { url = "https://files.pythonhosted.org/packages/e6/2d/bdc466a3db9145e946762d52cd55b1385509d9f9004fec1c97bdc8debbfb/black-25.12.0-cp313-cp313-win_arm64.whl", hash = "sha256:bc5b1c09fe3c931ddd20ee548511c64ebf964ada7e6f0763d443947fd1c603ce", size = 1239350, upload-time = "2025-12-08T01:46:09.458Z" },
- { url = "https://files.pythonhosted.org/packages/35/46/1d8f2542210c502e2ae1060b2e09e47af6a5e5963cb78e22ec1a11170b28/black-25.12.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0a0953b134f9335c2434864a643c842c44fba562155c738a2a37a4d61f00cad5", size = 1917015, upload-time = "2025-12-08T01:53:27.987Z" },
- { url = "https://files.pythonhosted.org/packages/41/37/68accadf977672beb8e2c64e080f568c74159c1aaa6414b4cd2aef2d7906/black-25.12.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2355bbb6c3b76062870942d8cc450d4f8ac71f9c93c40122762c8784df49543f", size = 1741830, upload-time = "2025-12-08T01:54:36.861Z" },
- { url = "https://files.pythonhosted.org/packages/ac/76/03608a9d8f0faad47a3af3a3c8c53af3367f6c0dd2d23a84710456c7ac56/black-25.12.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9678bd991cc793e81d19aeeae57966ee02909877cb65838ccffef24c3ebac08f", size = 1791450, upload-time = "2025-12-08T01:44:52.581Z" },
- { url = "https://files.pythonhosted.org/packages/06/99/b2a4bd7dfaea7964974f947e1c76d6886d65fe5d24f687df2d85406b2609/black-25.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:97596189949a8aad13ad12fcbb4ae89330039b96ad6742e6f6b45e75ad5cfd83", size = 1452042, upload-time = "2025-12-08T01:46:13.188Z" },
- { url = "https://files.pythonhosted.org/packages/b2/7c/d9825de75ae5dd7795d007681b752275ea85a1c5d83269b4b9c754c2aaab/black-25.12.0-cp314-cp314-win_arm64.whl", hash = "sha256:778285d9ea197f34704e3791ea9404cd6d07595745907dd2ce3da7a13627b29b", size = 1267446, upload-time = "2025-12-08T01:46:14.497Z" },
- { url = "https://files.pythonhosted.org/packages/68/11/21331aed19145a952ad28fca2756a1433ee9308079bd03bd898e903a2e53/black-25.12.0-py3-none-any.whl", hash = "sha256:48ceb36c16dbc84062740049eef990bb2ce07598272e673c17d1a7720c71c828", size = 206191, upload-time = "2025-12-08T01:40:50.963Z" },
-]
-
[[package]]
name = "certifi"
version = "2025.11.12"
@@ -931,37 +834,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },
]
-[[package]]
-name = "isort"
-version = "6.1.0"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
- "python_full_version < '3.10'",
-]
-dependencies = [
- { name = "importlib-metadata", marker = "python_full_version < '3.10'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/1e/82/fa43935523efdfcce6abbae9da7f372b627b27142c3419fcf13bf5b0c397/isort-6.1.0.tar.gz", hash = "sha256:9b8f96a14cfee0677e78e941ff62f03769a06d412aabb9e2a90487b3b7e8d481", size = 824325, upload-time = "2025-10-01T16:26:45.027Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/7f/cc/9b681a170efab4868a032631dea1e8446d8ec718a7f657b94d49d1a12643/isort-6.1.0-py3-none-any.whl", hash = "sha256:58d8927ecce74e5087aef019f778d4081a3b6c98f15a80ba35782ca8a2097784", size = 94329, upload-time = "2025-10-01T16:26:43.291Z" },
-]
-
-[[package]]
-name = "isort"
-version = "7.0.0"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
- "python_full_version >= '3.14'",
- "python_full_version == '3.13.*'",
- "python_full_version == '3.12.*'",
- "python_full_version == '3.11.*'",
- "python_full_version == '3.10.*'",
-]
-sdist = { url = "https://files.pythonhosted.org/packages/63/53/4f3c058e3bace40282876f9b553343376ee687f3c35a525dc79dbd450f88/isort-7.0.0.tar.gz", hash = "sha256:5513527951aadb3ac4292a41a16cbc50dd1642432f5e8c20057d414bdafb4187", size = 805049, upload-time = "2025-10-11T13:30:59.107Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/7f/ed/e3705d6d02b4f7aea715a353c8ce193efd0b5db13e204df895d38734c244/isort-7.0.0-py3-none-any.whl", hash = "sha256:1bcabac8bc3c36c7fb7b98a76c8abb18e0f841a3ba81decac7691008592499c1", size = 94672, upload-time = "2025-10-11T13:30:57.665Z" },
-]
-
[[package]]
name = "jinja2"
version = "3.1.6"
@@ -1830,16 +1702,12 @@ wheels = [
[[package]]
name = "openviking"
-version = "0.1.2"
+version = "0.1.6"
source = { editable = "." }
dependencies = [
{ name = "apscheduler" },
- { name = "black", version = "25.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
- { name = "black", version = "25.12.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
{ name = "fastapi" },
{ name = "httpx" },
- { name = "isort", version = "6.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
- { name = "isort", version = "7.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
{ name = "json-repair", version = "0.44.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
{ name = "json-repair", version = "0.55.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
{ name = "markdownify" },
@@ -1878,10 +1746,8 @@ dependencies = [
[package.metadata]
requires-dist = [
{ name = "apscheduler", specifier = ">=3.11.0" },
- { name = "black", specifier = ">=23.0.0" },
{ name = "fastapi", specifier = ">=0.128.0" },
{ name = "httpx", specifier = ">=0.25.0" },
- { name = "isort", specifier = ">=5.12.0" },
{ name = "json-repair", specifier = ">=0.25.0" },
{ name = "markdownify", specifier = ">=0.11.0" },
{ name = "mypy", specifier = ">=1.0.0" },
@@ -2341,34 +2207,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/2d/71/64e9b1c7f04ae0027f788a248e6297d7fcc29571371fe7d45495a78172c0/pillow-12.1.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:75af0b4c229ac519b155028fa1be632d812a519abba9b46b20e50c6caa184f19", size = 7029809, upload-time = "2026-01-02T09:13:26.541Z" },
]
-[[package]]
-name = "platformdirs"
-version = "4.4.0"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
- "python_full_version < '3.10'",
-]
-sdist = { url = "https://files.pythonhosted.org/packages/23/e8/21db9c9987b0e728855bd57bff6984f67952bea55d6f75e055c46b5383e8/platformdirs-4.4.0.tar.gz", hash = "sha256:ca753cf4d81dc309bc67b0ea38fd15dc97bc30ce419a7f58d13eb3bf14c4febf", size = 21634, upload-time = "2025-08-26T14:32:04.268Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/40/4b/2028861e724d3bd36227adfa20d3fd24c3fc6d52032f4a93c133be5d17ce/platformdirs-4.4.0-py3-none-any.whl", hash = "sha256:abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85", size = 18654, upload-time = "2025-08-26T14:32:02.735Z" },
-]
-
-[[package]]
-name = "platformdirs"
-version = "4.5.1"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
- "python_full_version >= '3.14'",
- "python_full_version == '3.13.*'",
- "python_full_version == '3.12.*'",
- "python_full_version == '3.11.*'",
- "python_full_version == '3.10.*'",
-]
-sdist = { url = "https://files.pythonhosted.org/packages/cf/86/0248f086a84f01b37aaec0fa567b397df1a119f73c16f6c7a9aac73ea309/platformdirs-4.5.1.tar.gz", hash = "sha256:61d5cdcc6065745cdd94f0f878977f8de9437be93de97c1c12f853c9c0cdcbda", size = 21715, upload-time = "2025-12-05T13:52:58.638Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/cb/28/3bfe2fa5a7b9c46fe7e13c97bda14c895fb10fa2ebf1d0abb90e0cea7ee1/platformdirs-4.5.1-py3-none-any.whl", hash = "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31", size = 18731, upload-time = "2025-12-05T13:52:56.823Z" },
-]
-
[[package]]
name = "pluggy"
version = "1.6.0"
@@ -2789,15 +2627,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },
]
-[[package]]
-name = "pytokens"
-version = "0.3.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/4e/8d/a762be14dae1c3bf280202ba3172020b2b0b4c537f94427435f19c413b72/pytokens-0.3.0.tar.gz", hash = "sha256:2f932b14ed08de5fcf0b391ace2642f858f1394c0857202959000b68ed7a458a", size = 17644, upload-time = "2025-11-05T13:36:35.34Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/84/25/d9db8be44e205a124f6c98bc0324b2bb149b7431c53877fc6d1038dddaf5/pytokens-0.3.0-py3-none-any.whl", hash = "sha256:95b2b5eaf832e469d141a378872480ede3f251a5a5041b8ec6e581d3ac71bbf3", size = 12195, upload-time = "2025-11-05T13:36:33.183Z" },
-]
-
[[package]]
name = "pytz"
version = "2025.2"