Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat: update media parsers (#196)

MaojiaShengdc7bc956f37f
diff --git a/crates/ov_cli/src/main.rs b/crates/ov_cli/src/main.rs
index 76d50f5c4..8ce253844 100644
--- a/crates/ov_cli/src/main.rs
+++ b/crates/ov_cli/src/main.rs
@@ -471,7 +471,7 @@ async fn main() {
}
async fn handle_add_resource(
- path: String,
+ mut path: String,
to: Option<String>,
reason: String,
instruction: String,
@@ -479,6 +479,34 @@ async fn handle_add_resource(
timeout: Option<f64>,
ctx: CliContext,
) -> Result<()> {
+ // Validate path: if it's a local path, check if it exists
+ if !path.starts_with("http://") && !path.starts_with("https://") {
+ use std::path::Path;
+
+ // Unescape path: replace backslash followed by space with just space
+ let unescaped_path = path.replace("\\ ", " ");
+ let path_obj = Path::new(&unescaped_path);
+ if !path_obj.exists() {
+ eprintln!("Error: Path '{}' does not exist.", path);
+
+ // Check if there might be unquoted spaces
+ use std::env;
+ let args: Vec<String> = env::args().collect();
+
+ if let Some(add_resource_pos) = args.iter().position(|s| s == "add-resource" || s == "add") {
+ if args.len() > add_resource_pos + 2 {
+ let extra_args = &args[add_resource_pos + 2..];
+ let suggested_path = format!("{} {}", path, extra_args.join(" "));
+ eprintln!("\nIt looks like you may have forgotten to quote a path with spaces.");
+ eprintln!("Suggested command: ov add-resource \"{}\"", suggested_path);
+ }
+ }
+
+ std::process::exit(1);
+ }
+ path = unescaped_path;
+ }
+
let client = ctx.get_client();
commands::resources::add_resource(
&client, &path, to, reason, instruction, wait, timeout, ctx.output_format, ctx.compact
diff --git a/examples/chatmem/ov.conf.example b/examples/chatmem/ov.conf.example
index 2e9a40ae1..6a085e5d9 100644
--- a/examples/chatmem/ov.conf.example
+++ b/examples/chatmem/ov.conf.example
@@ -12,6 +12,7 @@
"api_base" : "https://ark-cn-beijing.bytedance.net/api/v3",
"api_key" : "not_gonna_give_u_this",
"backend" : "volcengine",
- "model" : "doubao-seed-1-8-251228"
+ "model" : "doubao-seed-1-8-251228",
+ "thinking": false
}
}
diff --git a/examples/mcp-query/ov.conf.example b/examples/mcp-query/ov.conf.example
index bf4a45de8..fc40ea925 100644
--- a/examples/mcp-query/ov.conf.example
+++ b/examples/mcp-query/ov.conf.example
@@ -12,6 +12,7 @@
"api_base" : "https://ark-cn-beijing.bytedance.net/api/v3",
"api_key" : "<your-api-key>",
"provider" : "volcengine",
- "model" : "doubao-seed-1-8-251228"
+ "model" : "doubao-seed-1-8-251228",
+ "thinking": false
}
}
diff --git a/examples/memex/ov.conf.example b/examples/memex/ov.conf.example
index d5187cd33..42697b72d 100644
--- a/examples/memex/ov.conf.example
+++ b/examples/memex/ov.conf.example
@@ -12,6 +12,7 @@
"api_base" : "https://ark.cn-beijing.volces.com/api/v3",
"api_key" : "your-volcengine-api-key",
"backend" : "volcengine",
- "model" : "doubao-seed-1-8-251228"
+ "model" : "doubao-seed-1-8-251228",
+ "thinking": false
}
}
diff --git a/examples/ov.conf.example b/examples/ov.conf.example
index 205cd7d99..34cbc6a4a 100644
--- a/examples/ov.conf.example
+++ b/examples/ov.conf.example
@@ -45,7 +45,8 @@
"api_base": "https://ark.cn-beijing.volces.com/api/v3",
"temperature": 0.0,
"max_retries": 2,
- "provider": "volcengine"
+ "provider": "volcengine",
+ "thinking": false
},
"rerank": {
"ak": null,
diff --git a/examples/query/ov.conf.example b/examples/query/ov.conf.example
index 58d034c03..fdc7cb553 100644
--- a/examples/query/ov.conf.example
+++ b/examples/query/ov.conf.example
@@ -12,6 +12,7 @@
"api_base" : "https://ark-cn-beijing.bytedance.net/api/v3",
"api_key" : "not_gonna_give_u_this",
"provider" : "volcengine",
- "model" : "doubao-seed-1-8-251228"
+ "model" : "doubao-seed-1-8-251228",
+ "thinking": false
}
}
diff --git a/examples/server_client/ov.conf.example b/examples/server_client/ov.conf.example
index 582d79b80..13eb55db6 100644
--- a/examples/server_client/ov.conf.example
+++ b/examples/server_client/ov.conf.example
@@ -34,6 +34,7 @@
"api_base": "https://ark.cn-beijing.volces.com/api/v3",
"temperature": 0.0,
"max_retries": 2,
- "provider": "volcengine"
+ "provider": "volcengine",
+ "thinking": false
}
}
diff --git a/openviking/models/vlm/backends/litellm_vlm.py b/openviking/models/vlm/backends/litellm_vlm.py
index 0c9a7ab68..b31579c50 100644
--- a/openviking/models/vlm/backends/litellm_vlm.py
+++ b/openviking/models/vlm/backends/litellm_vlm.py
@@ -146,21 +146,31 @@ class LiteLLMVLMProvider(VLMBase):
return kwargs
- def get_completion(self, prompt: str) -> str:
+ def get_completion(self, prompt: str, thinking: bool = False) -> str:
"""Get text completion synchronously."""
model = self._resolve_model(self.model or "gpt-4o-mini")
messages = [{"role": "user", "content": prompt}]
+ original_thinking = self._thinking
+ if thinking:
+ self._thinking = thinking
kwargs = self._build_kwargs(model, messages)
+ self._thinking = original_thinking
response = completion(**kwargs)
self._update_token_usage_from_response(response)
return response.choices[0].message.content or ""
- async def get_completion_async(self, prompt: str, max_retries: int = 0) -> str:
+ async def get_completion_async(
+ self, prompt: str, thinking: bool = False, max_retries: int = 0
+ ) -> str:
"""Get text completion asynchronously."""
model = self._resolve_model(self.model or "gpt-4o-mini")
messages = [{"role": "user", "content": prompt}]
+ original_thinking = self._thinking
+ if thinking:
+ self._thinking = thinking
kwargs = self._build_kwargs(model, messages)
+ self._thinking = original_thinking
last_error = None
for attempt in range(max_retries + 1):
@@ -171,7 +181,7 @@ class LiteLLMVLMProvider(VLMBase):
except Exception as e:
last_error = e
if attempt < max_retries:
- await asyncio.sleep(2 ** attempt)
+ await asyncio.sleep(2**attempt)
if last_error:
raise last_error
@@ -181,6 +191,7 @@ class LiteLLMVLMProvider(VLMBase):
self,
prompt: str,
images: List[Union[str, Path, bytes]],
+ thinking: bool = False,
) -> str:
"""Get vision completion synchronously."""
model = self._resolve_model(self.model or "gpt-4o-mini")
@@ -191,7 +202,11 @@ class LiteLLMVLMProvider(VLMBase):
content.append({"type": "text", "text": prompt})
messages = [{"role": "user", "content": content}]
+ original_thinking = self._thinking
+ if thinking:
+ self._thinking = thinking
kwargs = self._build_kwargs(model, messages)
+ self._thinking = original_thinking
response = completion(**kwargs)
self._update_token_usage_from_response(response)
@@ -201,6 +216,7 @@ class LiteLLMVLMProvider(VLMBase):
self,
prompt: str,
images: List[Union[str, Path, bytes]],
+ thinking: bool = False,
) -> str:
"""Get vision completion asynchronously."""
model = self._resolve_model(self.model or "gpt-4o-mini")
@@ -211,7 +227,11 @@ class LiteLLMVLMProvider(VLMBase):
content.append({"type": "text", "text": prompt})
messages = [{"role": "user", "content": content}]
+ original_thinking = self._thinking
+ if thinking:
+ self._thinking = thinking
kwargs = self._build_kwargs(model, messages)
+ self._thinking = original_thinking
response = await acompletion(**kwargs)
self._update_token_usage_from_response(response)
diff --git a/openviking/models/vlm/backends/openai_vlm.py b/openviking/models/vlm/backends/openai_vlm.py
index c6c5b230e..d6f6effa9 100644
--- a/openviking/models/vlm/backends/openai_vlm.py
+++ b/openviking/models/vlm/backends/openai_vlm.py
@@ -61,9 +61,6 @@ class OpenAIVLM(VLMBase):
"temperature": self.temperature,
}
- if self.provider == "volcengine":
- kwargs["thinking"] = {"type": "disabled" if not thinking else "enabled"}
-
response = client.chat.completions.create(**kwargs)
self._update_token_usage_from_response(response)
return response.choices[0].message.content or ""
@@ -79,9 +76,6 @@ class OpenAIVLM(VLMBase):
"temperature": self.temperature,
}
- if self.provider == "volcengine":
- kwargs["thinking"] = {"type": "disabled" if not thinking else "enabled"}
-
last_error = None
for attempt in range(max_retries + 1):
try:
@@ -131,6 +125,7 @@ class OpenAIVLM(VLMBase):
self,
prompt: str,
images: List[Union[str, Path, bytes]],
+ thinking: bool = False,
) -> str:
"""Get vision completion"""
client = self.get_client()
@@ -140,11 +135,13 @@ class OpenAIVLM(VLMBase):
content.append(self._prepare_image(img))
content.append({"type": "text", "text": prompt})
- response = client.chat.completions.create(
- model=self.model or "gpt-4o-mini",
- messages=[{"role": "user", "content": content}],
- temperature=self.temperature,
- )
+ kwargs = {
+ "model": self.model or "gpt-4o-mini",
+ "messages": [{"role": "user", "content": content}],
+ "temperature": self.temperature,
+ }
+
+ response = client.chat.completions.create(**kwargs)
self._update_token_usage_from_response(response)
return response.choices[0].message.content or ""
@@ -152,6 +149,7 @@ class OpenAIVLM(VLMBase):
self,
prompt: str,
images: List[Union[str, Path, bytes]],
+ thinking: bool = False,
) -> str:
"""Get vision completion asynchronously"""
client = self.get_async_client()
@@ -161,10 +159,12 @@ class OpenAIVLM(VLMBase):
content.append(self._prepare_image(img))
content.append({"type": "text", "text": prompt})
- response = await client.chat.completions.create(
- model=self.model or "gpt-4o-mini",
- messages=[{"role": "user", "content": content}],
- temperature=self.temperature,
- )
+ kwargs = {
+ "model": self.model or "gpt-4o-mini",
+ "messages": [{"role": "user", "content": content}],
+ "temperature": self.temperature,
+ }
+
+ response = await client.chat.completions.create(**kwargs)
self._update_token_usage_from_response(response)
return response.choices[0].message.content or ""
diff --git a/openviking/models/vlm/backends/volcengine_vlm.py b/openviking/models/vlm/backends/volcengine_vlm.py
index f11a289e6..e4c4d2909 100644
--- a/openviking/models/vlm/backends/volcengine_vlm.py
+++ b/openviking/models/vlm/backends/volcengine_vlm.py
@@ -2,6 +2,8 @@
# SPDX-License-Identifier: Apache-2.0
"""VolcEngine VLM backend implementation"""
+import asyncio
+import base64
from pathlib import Path
from typing import Any, Dict, List, Union
@@ -54,22 +56,123 @@ class VolcEngineVLM(OpenAIVLM):
)
return self._async_client
- def get_completion(self, prompt: str) -> str:
- return super().get_completion(prompt)
+ def get_completion(self, prompt: str, thinking: bool = False) -> str:
+ """Get text completion"""
+ client = self.get_client()
+ kwargs = {
+ "model": self.model or "doubao-seed-1-8-251228",
+ "messages": [{"role": "user", "content": prompt}],
+ "temperature": self.temperature,
+ "thinking": {"type": "disabled" if not thinking else "enabled"},
+ }
- async def get_completion_async(self, prompt: str, max_retries: int = 0) -> str:
- return await super().get_completion_async(prompt, max_retries)
+ response = client.chat.completions.create(**kwargs)
+ self._update_token_usage_from_response(response)
+ return response.choices[0].message.content or ""
+
+ async def get_completion_async(
+ self, prompt: str, thinking: bool = False, max_retries: int = 0
+ ) -> str:
+ """Get text completion asynchronously"""
+ client = self.get_async_client()
+ kwargs = {
+ "model": self.model or "doubao-seed-1-8-251228",
+ "messages": [{"role": "user", "content": prompt}],
+ "temperature": self.temperature,
+ "thinking": {"type": "disabled" if not thinking else "enabled"},
+ }
+
+ last_error = None
+ for attempt in range(max_retries + 1):
+ try:
+ response = await client.chat.completions.create(**kwargs)
+ self._update_token_usage_from_response(response)
+ return response.choices[0].message.content or ""
+ except Exception as e:
+ last_error = e
+ if attempt < max_retries:
+ await asyncio.sleep(2**attempt)
+
+ if last_error:
+ raise last_error
+ else:
+ raise RuntimeError("Unknown error in async completion")
+
+ def _prepare_image(self, image: Union[str, Path, bytes]) -> Dict[str, Any]:
+ """Prepare image data"""
+ if isinstance(image, bytes):
+ b64 = base64.b64encode(image).decode("utf-8")
+ return {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/png;base64,{b64}"},
+ }
+ elif isinstance(image, Path) or (
+ isinstance(image, str) and not image.startswith(("http://", "https://"))
+ ):
+ path = Path(image)
+ suffix = path.suffix.lower()
+ mime_type = {
+ ".png": "image/png",
+ ".jpg": "image/jpeg",
+ ".jpeg": "image/jpeg",
+ ".gif": "image/gif",
+ ".webp": "image/webp",
+ }.get(suffix, "image/png")
+ with open(path, "rb") as f:
+ b64 = base64.b64encode(f.read()).decode("utf-8")
+ return {
+ "type": "image_url",
+ "image_url": {"url": f"data:{mime_type};base64,{b64}"},
+ }
+ else:
+ return {"type": "image_url", "image_url": {"url": image}}
def get_vision_completion(
self,
prompt: str,
images: List[Union[str, Path, bytes]],
+ thinking: bool = False,
) -> str:
- return super().get_vision_completion(prompt, images)
+ """Get vision completion"""
+ client = self.get_client()
+
+ content = []
+ for img in images:
+ content.append(self._prepare_image(img))
+ content.append({"type": "text", "text": prompt})
+
+ kwargs = {
+ "model": self.model or "doubao-seed-1-8-251228",
+ "messages": [{"role": "user", "content": content}],
+ "temperature": self.temperature,
+ "thinking": {"type": "disabled" if not thinking else "enabled"},
+ }
+
+ response = client.chat.completions.create(**kwargs)
+ self._update_token_usage_from_response(response)
+ return response.choices[0].message.content or ""
async def get_vision_completion_async(
self,
prompt: str,
images: List[Union[str, Path, bytes]],
+ thinking: bool = False,
) -> str:
- return await super().get_vision_completion_async(prompt, images)
+ """Get vision completion asynchronously"""
+ client = self.get_async_client()
+
+ content = []
+ for img in images:
+ content.append(self._prepare_image(img))
+ content.append({"type": "text", "text": prompt})
+
+ kwargs = {
+ "model": self.model or "doubao-seed-1-8-251228",
+ "messages": [{"role": "user", "content": content}],
+ "temperature": self.temperature,
+ "thinking": {"type": "disabled" if not thinking else "enabled"},
+ }
+
+ response = await client.chat.completions.create(**kwargs)
+ self._update_token_usage_from_response(response)
+ return response.choices[0].message.content or ""
diff --git a/openviking/models/vlm/base.py b/openviking/models/vlm/base.py
index ef55f7127..cd563f9c3 100644
--- a/openviking/models/vlm/base.py
+++ b/openviking/models/vlm/base.py
@@ -27,12 +27,14 @@ class VLMBase(ABC):
self._token_tracker = TokenUsageTracker()
@abstractmethod
- def get_completion(self, prompt: str) -> str:
+ def get_completion(self, prompt: str, thinking: bool = False) -> str:
"""Get text completion"""
pass
@abstractmethod
- async def get_completion_async(self, prompt: str, max_retries: int = 0) -> str:
+ async def get_completion_async(
+ self, prompt: str, thinking: bool = False, max_retries: int = 0
+ ) -> str:
"""Get text completion asynchronously"""
pass
@@ -41,6 +43,7 @@ class VLMBase(ABC):
self,
prompt: str,
images: List[Union[str, Path, bytes]],
+ thinking: bool = False,
) -> str:
"""Get vision completion"""
pass
@@ -50,6 +53,7 @@ class VLMBase(ABC):
self,
prompt: str,
images: List[Union[str, Path, bytes]],
+ thinking: bool = False,
) -> str:
"""Get vision completion asynchronously"""
pass
@@ -128,16 +132,20 @@ class VLMFactory:
if not use_litellm:
if provider == "openai":
from .backends.openai_vlm import OpenAIVLM
+
return OpenAIVLM(config)
elif provider == "volcengine":
from .backends.volcengine_vlm import VolcEngineVLM
+
return VolcEngineVLM(config)
from .backends.litellm_vlm import LiteLLMVLMProvider
+
return LiteLLMVLMProvider(config)
@staticmethod
def get_available_providers() -> List[str]:
"""Get list of available providers"""
from .registry import get_all_provider_names
+
return get_all_provider_names()
diff --git a/openviking/models/vlm/llm.py b/openviking/models/vlm/llm.py
index 07c52179f..6c8b9c569 100644
--- a/openviking/models/vlm/llm.py
+++ b/openviking/models/vlm/llm.py
@@ -168,34 +168,38 @@ class StructuredVLM:
self,
prompt: str,
schema: Optional[Dict[str, Any]] = None,
+ thinking: bool = False,
) -> Optional[Dict[str, Any]]:
"""Get JSON completion from VLM."""
if schema:
prompt = f"{prompt}\n\n{get_json_schema_prompt(schema)}"
- response = self._get_vlm().get_completion(prompt)
+ response = self._get_vlm().get_completion(prompt, thinking)
return parse_json_from_response(response)
async def complete_json_async(
self,
prompt: str,
schema: Optional[Dict[str, Any]] = None,
+ thinking: bool = False,
+ max_retries: int = 0,
) -> Optional[Dict[str, Any]]:
"""Async version of complete_json."""
if schema:
prompt = f"{prompt}\n\n{get_json_schema_prompt(schema)}"
- response = await self._get_vlm().get_completion_async(prompt)
+ response = await self._get_vlm().get_completion_async(prompt, thinking, max_retries)
return parse_json_from_response(response)
def complete_model(
self,
prompt: str,
model_class: Type[T],
+ thinking: bool = False,
) -> Optional[T]:
"""Get structured completion validated against a Pydantic model."""
schema = model_class.model_json_schema()
- response = self.complete_json(prompt, schema=schema)
+ response = self.complete_json(prompt, schema=schema, thinking=thinking)
if response is None:
return None
@@ -209,10 +213,14 @@ class StructuredVLM:
self,
prompt: str,
model_class: Type[T],
+ thinking: bool = False,
+ max_retries: int = 0,
) -> Optional[T]:
"""Async version of complete_model."""
schema = model_class.model_json_schema()
- response = await self.complete_json_async(prompt, schema=schema)
+ response = await self.complete_json_async(
+ prompt, schema=schema, thinking=thinking, max_retries=max_retries
+ )
if response is None:
return None
@@ -226,14 +234,16 @@ class StructuredVLM:
self,
prompt: str,
images: list,
+ thinking: bool = False,
) -> str:
"""Get vision completion."""
- return self._get_vlm().get_vision_completion(prompt, images)
+ return self._get_vlm().get_vision_completion(prompt, images, thinking)
async def get_vision_completion_async(
self,
prompt: str,
images: list,
+ thinking: bool = False,
) -> str:
"""Async vision completion."""
- return await self._get_vlm().get_vision_completion_async(prompt, images)
+ return await self._get_vlm().get_vision_completion_async(prompt, images, thinking)
diff --git a/openviking/parse/base.py b/openviking/parse/base.py
index 9548f4b4c..91eec9009 100644
--- a/openviking/parse/base.py
+++ b/openviking/parse/base.py
@@ -254,7 +254,7 @@ class ResourceNode:
texts.append(child.get_text(include_children=True))
return "\n".join(texts)
- def get_abstract(self, max_length: int = 200) -> str:
+ def get_abstract(self, max_length: int = 256) -> str:
"""
Generate L0 abstract for this node.
@@ -264,6 +264,8 @@ class ResourceNode:
Returns:
Abstract text
"""
+ if "abstract" in self.meta:
+ return self.meta["abstract"]
if self.title:
abstract = self.title
else:
@@ -285,8 +287,10 @@ class ResourceNode:
Returns:
Overview text including structure summary
"""
+ if "overview" in self.meta:
+ return self.meta["overview"]
+ # Default overview generation
parts = []
-
if self.title:
parts.append(f"**{self.title}**")
diff --git a/openviking/parse/directory_scan.py b/openviking/parse/directory_scan.py
index 2b62b6c19..8da532f27 100644
--- a/openviking/parse/directory_scan.py
+++ b/openviking/parse/directory_scan.py
@@ -164,6 +164,7 @@ def _classify_file(
Processable: ParserRegistry has a parser, or is_text_file (code/config/docs).
"""
+ # Normal classification logic
if registry.get_parser_for_file(file_path) is not None:
return CLASS_PROCESSABLE
if is_text_file(file_path):
diff --git a/openviking/parse/parsers/README.md b/openviking/parse/parsers/README.md
index b8bcefe0b..ceffac297 100644
--- a/openviking/parse/parsers/README.md
+++ b/openviking/parse/parsers/README.md
@@ -143,10 +143,68 @@ L1: """
代码解析器,支持语法高亮和代码结构分析。能识别函数、类、方法等代码元素。
-### 6. MediaParser (`media.py`)
-**支持格式**: `.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, `.mp4`, `.mov`, `.avi`, `.webm`, `.mp3`, `.wav`, `.m4a`, `.flac`
+### 6. MediaParser (`media/*.py`)
+
+**支持格式**:
+- 图片: `.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`
+- 视频: `.mp4`, `.mov`, `.avi`, `.webm`
+- 音频: `.mp3`, `.wav`, `.m4a`, `.flac`
+
+多媒体解析器,使用 VLM(视觉语言模型)分析图像、视频和音频内容,生成文本描述。多媒体解析器当且仅当 add-resource 调用时只添加上述文件类型时生效。即:
+1. 当添加目录时,系统将对多媒体文件暂不生成单独目录和文本描述,仅存储和进行递归摘要。
+2. 当单独添加多媒体文件时,多媒体解析器会直接解析该文件,然后通过单独目录存放,在目录下生成文本描述。
+
+#### 存储组织策略
+
+多媒体文件的存储采用以下层级结构:
+
+```
+viking://resource/
+├── images/ # 图片文件
+│ └── 20240820/ # 上传日期(YYYYMMDD)
+│ └── 20240820_123456_jpg/ # 文件文件夹(文件名_扩展名)
+│ ├── .abstract.md # L0 摘要
+│ ├── .overview.md # L1 概览
+│ └── 20240820_123456.jpg # 原始文件
+├── audio/ # 音频文件
+│ └── 20240820/
+│ └── my_song_mp3/
+│ ├── .abstract.md
+│ ├── .overview.md
+│ └── my_song.mp3
+└── video/ # 视频文件
+ └── 20240820/
+ └── my_video_mp4/
+ ├── .abstract.md
+ ├── .overview.md
+ └── my_video.mp4
+```
+
+详细说明:
+
+1. **媒体子目录**: 在 `viking://resource` 下按类型划分为三个子目录
+ - `viking://resource/images`: 存储未明确指定目标路径的图片文件
+ - `viking://resource/audio`: 存储未明确指定目标路径的音频文件
+ - `viking://resource/video`: 存储未明确指定目标路径的视频文件
+
+2. **日期子目录**: 每次上传的文件按当前日期(格式:YYYYMMDD)组织,而非文件内部元信息的时间
+ - 例如:`viking://resource/images/20240820/` 存储 2024年8月20日上传的所有图片
+
+3. **文件文件夹**: 为每个多媒体文件创建一个专属文件夹,命名规则为:`文件名_扩展名`(扩展名不含点)
+ - 示例:上传 `20240820_123456.jpg` → 创建文件夹 `20240820_123456_jpg`
+
+4. **文件夹内容**: 每个文件文件夹内包含:
+ - **原始文件**: 保持原始文件名,空格字符替换为下划线 `_`(因为 OpenViking URI 不允许包含空格)
+ - 示例:`photo 1.jpg` → 保存为 `photo_1.jpg`
+ - `.abstract.md` (L0 层): 摘要信息(<200 token)
+ - 图片:文件名、内容描述、画面风格等
+ - 音频:文件名、时长、内容概述等
+ - 视频:文件名、时长、内容概述等
+ - `.overview.md` (L1 层): 概览信息
+ - 图片:除摘要外,还包含尺寸、OCR 识别结果、场景和主体描述等
+ - 音频:除摘要外,还包含语音/歌词识别结果、章节时间线等
+ - 视频:除摘要外,还包含使用场景等(未来会支持切分视频,递归存储子文件)
-多媒体解析器,使用 VLM(视觉语言模型)分析图像、视频和音频内容,生成文本描述。
## 核心组件
diff --git a/openviking/parse/parsers/directory.py b/openviking/parse/parsers/directory.py
index 5f2f05fa0..6fc6d8a62 100644
--- a/openviking/parse/parsers/directory.py
+++ b/openviking/parse/parsers/directory.py
@@ -27,6 +27,7 @@ from openviking.parse.base import (
create_parse_result,
)
from openviking.parse.parsers.base_parser import BaseParser
+from openviking.parse.parsers.media.constants import MEDIA_EXTENSIONS
from openviking_cli.utils.logger import get_logger
if TYPE_CHECKING:
@@ -74,7 +75,8 @@ class DirectoryParser(BaseParser):
source: Path to the directory.
instruction: Processing instruction (forwarded where applicable).
**kwargs: Extra options forwarded to ``scan_directory``:
- ``strict``, ``ignore_dirs``, ``include``, ``exclude``.
+ ``strict``, ``ignore_dirs``, ``include``, ``exclude``,
+ ``directly_upload_media``.
Returns:
``ParseResult`` with ``temp_dir_path`` pointing to VikingFS temp.
@@ -103,6 +105,7 @@ class DirectoryParser(BaseParser):
include=kwargs.get("include"),
exclude=kwargs.get("exclude"),
)
+ directly_upload_media = kwargs.get("directly_upload_media", True)
processable_files = scan_result.all_processable_files()
warnings.extend(scan_result.warnings)
@@ -137,13 +140,35 @@ class DirectoryParser(BaseParser):
for cf in processable_files:
file_parser = self._assign_parser(cf, registry)
parser_name = type(file_parser).__name__ if file_parser else "direct"
- ok = await self._process_single_file(
- cf,
- file_parser,
- target_uri,
- viking_fs,
- warnings,
- )
+
+ # Check if this is a media parser and we should directly upload
+ is_media_parser = file_parser and parser_name in [
+ "ImageParser",
+ "AudioParser",
+ "VideoParser",
+ ]
+ ext = Path(cf.path).suffix.lower()
+ is_media_file = ext in MEDIA_EXTENSIONS
+
+ if directly_upload_media and is_media_parser and is_media_file:
+ # Directly upload media file without using media parser
+ ok = await self._upload_file_directly(
+ cf,
+ target_uri,
+ viking_fs,
+ warnings,
+ )
+ parser_name = "direct_upload"
+ else:
+ # Normal processing with parser
+ ok = await self._process_single_file(
+ cf,
+ file_parser,
+ target_uri,
+ viking_fs,
+ warnings,
+ )
+
if ok:
file_count += 1
processed_files.append(
@@ -332,6 +357,32 @@ class DirectoryParser(BaseParser):
warnings.append(f"Failed to upload {rel_path}: {exc}")
return False
+ @staticmethod
+ async def _upload_file_directly(
+ classified_file: "ClassifiedFile",
+ target_uri: str,
+ viking_fs: Any,
+ warnings: List[str],
+ ) -> bool:
+ """Directly upload a file without using its parser.
+
+ Used for media files when directly_upload_media=True.
+
+ Returns:
+ *True* on success, *False* on failure.
+ """
+ rel_path = classified_file.rel_path
+ src_file = classified_file.path
+
+ try:
+ content = src_file.read_bytes()
+ dst_uri = f"{target_uri}/{rel_path}"
+ await viking_fs.write_file(dst_uri, content)
+ return True
+ except Exception as exc:
+ warnings.append(f"Failed to upload {rel_path}: {exc}")
+ return False
+
# ------------------------------------------------------------------
# VikingFS merge helpers
# ------------------------------------------------------------------
diff --git a/openviking/parse/parsers/html.py b/openviking/parse/parsers/html.py
index 34711ef0c..3b9739fbb 100644
--- a/openviking/parse/parsers/html.py
+++ b/openviking/parse/parsers/html.py
@@ -602,7 +602,11 @@ class HTMLParser(BaseParser):
def _sanitize_for_path(self, text: str, max_length: int = 50) -> str:
"""Sanitize text for use in file path, hash & shorten if too long."""
- safe = re.sub(r"[^\w\u4e00-\u9fff\s-]", "", text)
+ safe = re.sub(
+ r"[^\w\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af\u3400-\u4dbf\U00020000-\U0002a6df\s-]",
+ "",
+ text,
+ )
safe = re.sub(r"\s+", "_", safe)
safe = safe.strip("_")
if not safe:
diff --git a/openviking/parse/parsers/markdown.py b/openviking/parse/parsers/markdown.py
index bcb4f48ae..60ae999b2 100644
--- a/openviking/parse/parsers/markdown.py
+++ b/openviking/parse/parsers/markdown.py
@@ -335,7 +335,11 @@ class MarkdownParser(BaseParser):
return parts if parts else [content]
def _sanitize_for_path(self, text: str, max_length: int = 50) -> str:
- safe = re.sub(r"[^\w\u4e00-\u9fff\s-]", "", text)
+ safe = re.sub(
+ r"[^\w\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af\u3400-\u4dbf\U00020000-\U0002a6df\s-]",
+ "",
+ text,
+ )
safe = re.sub(r"\s+", "_", safe)
safe = safe.strip("_")
if not safe:
diff --git a/openviking/parse/parsers/media.py b/openviking/parse/parsers/media.py
deleted file mode 100644
index cb25a8036..000000000
--- a/openviking/parse/parsers/media.py
+++ /dev/null
@@ -1,536 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: Apache-2.0
-"""
-Media parser interfaces for OpenViking - Future expansion.
-
-This module defines parser interfaces for media types (image, audio, video).
-These are placeholder implementations that raise NotImplementedError.
-They serve as a design reference for future media parsing capabilities.
-
-For current document parsing (PDF, Markdown, HTML, Text), see other parser modules.
-"""
-
-from pathlib import Path
-from typing import List, Optional, Union
-
-from PIL import Image
-
-from openviking.parse.base import NodeType, ParseResult, ResourceNode
-from openviking.parse.parsers.base_parser import BaseParser
-from openviking_cli.utils.config.parser_config import AudioConfig, ImageConfig, VideoConfig
-
-# =============================================================================
-# Configuration Classes
-# =============================================================================
-
-
-# =============================================================================
-# Parser Classes
-# =============================================================================
-
-
-class ImageParser(BaseParser):
- """
- Image parser - Future implementation.
-
- Planned Features:
- 1. Visual content understanding using VLM (Vision Language Model)
- 2. OCR text extraction for images containing text
- 3. Metadata extraction (dimensions, format, EXIF data)
- 4. Generate semantic description and structured ResourceNode
-
- Example workflow:
- 1. Load image file
- 2. (Optional) Perform OCR to extract text
- 3. (Optional) Use VLM to generate visual description
- 4. Create ResourceNode with image metadata and descriptions
- 5. Return ParseResult
-
- Supported formats: PNG, JPG, JPEG, GIF, BMP, WEBP, SVG
- """
-
- def __init__(self, config: Optional[ImageConfig] = None, **kwargs):
- """
- Initialize ImageParser.
-
- Args:
- config: Image parsing configuration
- **kwargs: Additional configuration parameters
- """
- self.config = config or ImageConfig()
-
- @property
- def supported_extensions(self) -> List[str]:
- """Return supported image file extensions."""
- return [".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp", ".svg"]
-
- async def parse(self, source: Union[str, Path], instruction: str = "", **kwargs) -> ParseResult:
- """
- Parse image file using three-phase architecture.
-
- Phase 1: Generate temporary files
- - Copy original image to temp_uri/content.{ext}
- - Generate description.md using VLM
- - (Optional) Generate ocr.md using OCR
-
- Phase 2: Generate semantic info
- - Generate abstract and overview based on description.md
- - Overview includes file list and usage instructions
-
- Phase 3: Build directory structure
- - Move all files to final URI
- - Generate .abstract.md, .overview.md
-
- Args:
- source: Image file path
- **kwargs: Additional parsing parameters
-
- Returns:
- ParseResult with image content
-
- Raises:
- FileNotFoundError: If source file does not exist
- IOError: If image processing fails
- """
- from openviking.storage.viking_fs import get_viking_fs
-
- # Convert to Path object
- file_path = Path(source) if isinstance(source, str) else source
- if not file_path.exists():
- raise FileNotFoundError(f"Image file not found: {source}")
-
- viking_fs = get_viking_fs()
- temp_uri = viking_fs.create_temp_uri()
-
- # Phase 1: Generate temporary files
- image_bytes = file_path.read_bytes()
- ext = file_path.suffix
-
- # 1.1 Save original image
- await viking_fs.write_file_bytes(f"{temp_uri}/content{ext}", image_bytes)
-
- # 1.2 Extract image metadata
- try:
- img = Image.open(file_path)
- width, height = img.size
- format_str = img.format or ext[1:].upper()
- except Exception:
- width, height = 0, 0
- format_str = ext[1:].upper()
-
- # 1.3 Generate VLM description
- description = ""
- if self.config.enable_vlm:
- description = await self._vlm_describe(image_bytes, self.config.vlm_model)
- else:
- # Fallback: basic description
- description = f"Image file: {file_path.name} ({format_str}, {width}x{height})"
-
- await viking_fs.write_file(f"{temp_uri}/description.md", description)
-
- # 1.4 OCR (optional)
- ocr_text = None
- if self.config.enable_ocr:
- ocr_text = await self._ocr_extract(image_bytes, self.config.ocr_lang)
- if ocr_text:
- await viking_fs.write_file(f"{temp_uri}/ocr.md", ocr_text)
-
- # Create ResourceNode
- root_node = ResourceNode(
- type=NodeType.ROOT,
- title=file_path.stem,
- level=0,
- detail_file=None,
- content_path=None,
- children=[],
- meta={
- "width": width,
- "height": height,
- "format": format_str.lower(),
- "content_type": "image",
- "source_title": file_path.stem,
- "semantic_name": file_path.stem,
- },
- )
-
- # Phase 2: Generate semantic info
- await self._generate_semantic_info(root_node, temp_uri, viking_fs, ocr_text is not None)
-
- # Phase 3: Build directory structure (handled by TreeBuilder)
- return ParseResult(
- root=root_node,
- source_path=str(file_path),
- temp_dir_path=temp_uri,
- source_format="image",
- parser_name="ImageParser",
- meta={"content_type": "image", "format": format_str.lower()},
- )
-
- async def _vlm_describe(self, image_bytes: bytes, model: Optional[str]) -> str:
- """
- Generate image description using VLM.
-
- Args:
- image_bytes: Image binary data
- model: VLM model name
-
- Returns:
- Image description in markdown format
-
- TODO: Integrate with actual VLM API (OpenAI GPT-4V, Claude Vision, etc.)
- """
- # Fallback implementation - returns basic placeholder
- return "Image description (VLM integration pending)\n\nThis is an image. VLM description feature has not yet integrated external API."
-
- async def _ocr_extract(self, image_bytes: bytes, lang: str) -> Optional[str]:
- """
- Extract text from image using OCR.
-
- Args:
- image_bytes: Image binary data
- lang: OCR language code
-
- Returns:
- Extracted text in markdown format, or None if no text found
-
- TODO: Integrate with OCR API (Tesseract, PaddleOCR, etc.)
- """
- # Not implemented - return None
- return None
-
- async def _generate_semantic_info(
- self, node: ResourceNode, temp_uri: str, viking_fs, has_ocr: bool
- ):
- """
- Phase 2: Generate abstract and overview.
-
- Args:
- node: ResourceNode to update
- temp_uri: Temporary URI
- viking_fs: VikingFS instance
- has_ocr: Whether OCR file exists
- """
- # Read description.md
- description = await viking_fs.read_file(f"{temp_uri}/description.md")
-
- # Generate abstract (short summary, < 100 tokens)
- abstract = description[:200] if len(description) > 200 else description
-
- # Generate overview (content summary + file list + usage instructions)
- overview_parts = [
- "## Content Summary\n",
- description,
- "\n\n## Available Files\n",
- f"- content.{node.meta['format']}: Original image file ({node.meta['width']}x{node.meta['height']}, {node.meta['format'].upper()} format)\n",
- "- description.md: Detailed image description generated by VLM\n",
- ]
-
- if has_ocr:
- overview_parts.append("- ocr.md: OCR text recognition result from the image\n")
-
- overview_parts.append("\n## Usage\n")
- overview_parts.append("### View Image\n")
- overview_parts.append("```python\n")
- overview_parts.append("image_bytes = await image_resource.view()\n")
- overview_parts.append("# Returns: PNG/JPG format image binary data\n")
- overview_parts.append("# Purpose: Display or save the image\n")
- overview_parts.append("```\n\n")
-
- overview_parts.append("### Get VLM-generated Image Description\n")
- overview_parts.append("```python\n")
- overview_parts.append("description = await image_resource.description()\n")
- overview_parts.append("# Returns: FileContent object for further processing\n")
- overview_parts.append("# Purpose: Understand image content\n")
- overview_parts.append("```\n\n")
-
- if has_ocr:
- overview_parts.append("### Get OCR-recognized Text\n")
- overview_parts.append("```python\n")
- overview_parts.append("ocr_text = await image_resource.ocr()\n")
- overview_parts.append("# Returns: FileContent object or None\n")
- overview_parts.append("# Purpose: Extract text information from the image\n")
- overview_parts.append("```\n\n")
-
- overview_parts.append("### Get Image Metadata\n")
- overview_parts.append("```python\n")
- overview_parts.append(
- f"size = image_resource.get_size() # ({node.meta['width']}, {node.meta['height']})\n"
- )
- overview_parts.append(f'format = image_resource.get_format() # "{node.meta["format"]}"\n')
- overview_parts.append("```\n")
-
- overview = "".join(overview_parts)
-
- # Store in node meta
- node.meta["abstract"] = abstract
- node.meta["overview"] = overview
-
- async def parse_content(
- self, content: str, source_path: Optional[str] = None, instruction: str = "", **kwargs
- ) -> ParseResult:
- """
- Parse image from content string - Not yet implemented.
-
- Args:
- content: Image content (base64 or binary string)
- source_path: Optional source path for metadata
- **kwargs: Additional parsing parameters
-
- Returns:
- ParseResult with image content
-
- Raises:
- NotImplementedError: This feature is not yet implemented
- """
- raise NotImplementedError("Image parsing not yet implemented")
-
-
-class AudioParser(BaseParser):
- """
- Audio parser - Future implementation.
-
- Planned Features:
- 1. Speech-to-text transcription using ASR models
- 2. Audio metadata extraction (duration, sample rate, channels)
- 3. Speaker diarization (identify different speakers)
- 4. Timestamp alignment for transcribed text
- 5. Generate structured ResourceNode with transcript
-
- Example workflow:
- 1. Load audio file
- 2. Extract metadata (duration, format, sample rate)
- 3. Transcribe speech to text using Whisper or similar
- 4. (Optional) Perform speaker diarization
- 5. Create ResourceNode with:
- - type: NodeType.ROOT
- - children: sections for each speaker/timestamp
- - meta: audio metadata and timestamps
- 6. Return ParseResult
-
- Supported formats: MP3, WAV, OGG, FLAC, AAC, M4A
- """
-
- def __init__(self, config: Optional[AudioConfig] = None, **kwargs):
- """
- Initialize AudioParser.
-
- Args:
- config: Audio parsing configuration
- **kwargs: Additional configuration parameters
- """
- self.config = config or AudioConfig()
-
- @property
- def supported_extensions(self) -> List[str]:
- """Return supported audio file extensions."""
- return [".mp3", ".wav", ".ogg", ".flac", ".aac", ".m4a", ".opus"]
-
- async def parse(self, source: Union[str, Path], instruction: str = "", **kwargs) -> ParseResult:
- """
- Parse audio file - Not yet implemented.
-
- Planned implementation:
- 1. Load audio file
- 2. Extract metadata using librosa or similar
- 3. If enable_transcription:
- - Transcribe using Whisper or similar ASR model
- - Generate timestamps for each segment
- - (Optional) Perform speaker diarization
- 4. Create ResourceNode tree:
- - Root node with audio metadata
- - Child nodes for each transcribed segment
- 5. Return ParseResult
-
- Args:
- source: Audio file path or URL
- **kwargs: Additional parsing parameters
-
- Returns:
- ParseResult with transcribed content
-
- Raises:
- NotImplementedError: This feature is not yet implemented
- """
- raise NotImplementedError(
- "Audio parsing is not yet implemented. "
- "This is a placeholder interface for future expansion. "
- "\n\nPlanned features:"
- "\n- Speech-to-text transcription (Whisper)"
- "\n- Speaker diarization"
- "\n- Timestamp alignment"
- "\n- Audio metadata extraction"
- "\n\nWorkaround: Extract audio manually and add transcripts as "
- "text or markdown files."
- )
-
- async def parse_content(
- self, content: str, source_path: Optional[str] = None, instruction: str = "", **kwargs
- ) -> ParseResult:
- """
- Parse audio from content string - Not yet implemented.
-
- Args:
- content: Audio content (base64 or binary string)
- source_path: Optional source path for metadata
- **kwargs: Additional parsing parameters
-
- Returns:
- ParseResult with transcribed content
-
- Raises:
- NotImplementedError: This feature is not yet implemented
- """
- raise NotImplementedError("Audio parsing not yet implemented")
-
-
-class VideoParser(BaseParser):
- """
- Video parser - Future implementation.
-
- Planned Features:
- 1. Key frame extraction at regular intervals
- 2. Audio track transcription using ASR
- 3. VLM-based scene description for key frames
- 4. Video metadata extraction (duration, resolution, codec)
- 5. Generate structured ResourceNode combining visual and audio
-
- Example workflow:
- 1. Load video file
- 2. Extract metadata (duration, resolution, fps)
- 3. Extract audio track → transcribe using AudioParser
- 4. Extract key frames at specified intervals
- 5. For each frame: generate VLM description
- 6. Create ResourceNode tree:
- - Root: video metadata
- - Children: timeline nodes (each with frame + transcript)
- 7. Return ParseResult
-
- Supported formats: MP4, AVI, MOV, MKV, WEBM
- """
-
- def __init__(self, config: Optional[VideoConfig] = None, **kwargs):
- """
- Initialize VideoParser.
-
- Args:
- config: Video parsing configuration
- **kwargs: Additional configuration parameters
- """
- self.config = config or VideoConfig()
-
- @property
- def supported_extensions(self) -> List[str]:
- """Return supported video file extensions."""
- return [".mp4", ".avi", ".mov", ".mkv", ".webm", ".flv", ".wmv"]
-
- async def parse(self, source: Union[str, Path], instruction: str = "", **kwargs) -> ParseResult:
- """
- Parse video file - Not yet implemented.
-
- Planned implementation:
- 1. Load video file using cv2 or similar
- 2. Extract metadata (duration, resolution, fps, codec)
- 3. Extract audio track:
- - Save as temporary audio file
- - Parse using AudioParser
- 4. Extract key frames:
- - At specified intervals (e.g., every 10 seconds)
- - Save frames as images
- 5. For each frame (if enable_vlm_description):
- - Use VLM to generate scene description
- 6. Create ResourceNode tree:
- - Root: video metadata
- - Children: Timeline segments
- - Each segment contains:
- - Timestamp
- - Frame description (VLM)
- - Transcript (ASR)
- 7. Return ParseResult
-
- Args:
- source: Video file path or URL
- **kwargs: Additional parsing parameters
-
- Returns:
- ParseResult with video content
-
- Raises:
- NotImplementedError: This feature is not yet implemented
- """
- raise NotImplementedError(
- "Video parsing is not yet implemented. "
- "This is a placeholder interface for future expansion. "
- "\n\nPlanned features:"
- "\n- Key frame extraction"
- "\n- Audio track transcription"
- "\n- VLM scene description"
- "\n- Timeline-based structured output"
- "\n\nWorkaround: Extract frames and audio manually, then process "
- "as images and audio files."
- )
-
- async def parse_content(
- self, content: str, source_path: Optional[str] = None, instruction: str = "", **kwargs
- ) -> ParseResult:
- """
- Parse video from content string - Not yet implemented.
-
- Args:
- content: Video content (base64 or binary string)
- source_path: Optional source path for metadata
- **kwargs: Additional parsing parameters
-
- Returns:
- ParseResult with video content
-
- Raises:
- NotImplementedError: This feature is not yet implemented
- """
- raise NotImplementedError("Video parsing not yet implemented")
-
-
-# =============================================================================
-# Utility Functions
-# =============================================================================
-
-
-def is_media_parser_available(parser_type: str) -> bool:
- """
- Check if a media parser type is currently available.
-
- Args:
- parser_type: Type of parser ("image", "audio", "video")
-
- Returns:
- False (all media parsers are future implementations)
-
- Examples:
- >>> is_media_parser_available("image")
- False
- >>> is_media_parser_available("video")
- False
- """
- return False
-
-
-def get_media_parser_status() -> dict:
- """
- Get status of all media parsers.
-
- Returns:
- Dictionary with parser names and their implementation status
-
- Examples:
- >>> status = get_media_parser_status()
- >>> print(status)
- {
- "image": "planned",
- "audio": "planned",
- "video": "planned"
- }
- """
- return {
- "image": "planned",
- "audio": "planned",
- "video": "planned",
- }
diff --git a/openviking/parse/parsers/media/__init__.py b/openviking/parse/parsers/media/__init__.py
new file mode 100644
index 000000000..9f58977f4
--- /dev/null
+++ b/openviking/parse/parsers/media/__init__.py
@@ -0,0 +1,9 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+
+from .audio import AudioParser
+from .image import ImageParser
+from .utils import get_media_base_uri, get_media_type
+from .video import VideoParser
+
+__all__ = ["ImageParser", "AudioParser", "VideoParser", "get_media_type", "get_media_base_uri"]
diff --git a/openviking/parse/parsers/media/audio.py b/openviking/parse/parsers/media/audio.py
new file mode 100644
index 000000000..3caeb44a4
--- /dev/null
+++ b/openviking/parse/parsers/media/audio.py
@@ -0,0 +1,274 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+"""
+Audio parser - Future implementation.
+
+Planned Features:
+1. Speech-to-text transcription using ASR models
+2. Audio metadata extraction (duration, sample rate, channels)
+3. Speaker diarization (identify different speakers)
+4. Timestamp alignment for transcribed text
+5. Generate structured ResourceNode with transcript
+
+Example workflow:
+ 1. Load audio file
+ 2. Extract metadata (duration, format, sample rate)
+ 3. Transcribe speech to text using Whisper or similar
+ 4. (Optional) Perform speaker diarization
+ 5. Create ResourceNode with:
+ - type: NodeType.ROOT
+ - children: sections for each speaker/timestamp
+ - meta: audio metadata and timestamps
+ 6. Return ParseResult
+
+Supported formats: MP3, WAV, OGG, FLAC, AAC, M4A
+"""
+
+from pathlib import Path
+from typing import List, Optional, Union
+
+from openviking.parse.base import NodeType, ParseResult, ResourceNode
+from openviking.parse.parsers.base_parser import BaseParser
+from openviking.parse.parsers.media.constants import AUDIO_EXTENSIONS
+from openviking_cli.utils.config.parser_config import AudioConfig
+
+
+class AudioParser(BaseParser):
+ """
+ Audio parser for audio files.
+ """
+
+ def __init__(self, config: Optional[AudioConfig] = None, **kwargs):
+ """
+ Initialize AudioParser.
+
+ Args:
+ config: Audio parsing configuration
+ **kwargs: Additional configuration parameters
+ """
+ self.config = config or AudioConfig()
+
+ @property
+ def supported_extensions(self) -> List[str]:
+ """Return supported audio file extensions."""
+ return AUDIO_EXTENSIONS
+
+ async def parse(self, source: Union[str, Path], instruction: str = "", **kwargs) -> ParseResult:
+ """
+ Parse audio file - only copy original file and extract basic metadata, no content understanding.
+
+ Args:
+ source: Audio file path
+ **kwargs: Additional parsing parameters
+
+ Returns:
+ ParseResult with audio content
+
+ Raises:
+ FileNotFoundError: If source file does not exist
+ IOError: If audio processing fails
+ """
+ from openviking.storage.viking_fs import get_viking_fs
+
+ # Convert to Path object
+ file_path = Path(source) if isinstance(source, str) else source
+ if not file_path.exists():
+ raise FileNotFoundError(f"Audio file not found: {source}")
+
+ viking_fs = get_viking_fs()
+ temp_uri = viking_fs.create_temp_uri()
+
+ # Phase 1: Generate temporary files
+ audio_bytes = file_path.read_bytes()
+ ext = file_path.suffix
+
+ from openviking_cli.utils.uri import VikingURI
+
+ # Sanitize original filename (replace spaces with underscores)
+ original_filename = file_path.name.replace(" ", "_")
+ # Root directory name: filename stem + _ + extension (without dot)
+ stem = file_path.stem.replace(" ", "_")
+ ext_no_dot = ext[1:] if ext else ""
+ root_dir_name = VikingURI.sanitize_segment(f"{stem}_{ext_no_dot}")
+ root_dir_uri = f"{temp_uri}/{root_dir_name}"
+ await viking_fs.mkdir(root_dir_uri)
+
+ # 1.1 Save original audio with original filename (sanitized)
+ await viking_fs.write_file_bytes(f"{root_dir_uri}/{original_filename}", audio_bytes)
+
+ # 1.2 Validate audio file using magic bytes
+ # Define magic bytes for supported audio formats
+ audio_magic_bytes = {
+ ".mp3": [b"ID3", b"\xff\xfb", b"\xff\xf3", b"\xff\xf2"],
+ ".wav": [b"RIFF"],
+ ".ogg": [b"OggS"],
+ ".flac": [b"fLaC"],
+ ".aac": [b"\xff\xf1", b"\xff\xf9"],
+ ".m4a": [b"\x00\x00\x00", b"ftypM4A", b"ftypisom"],
+ ".opus": [b"OggS"],
+ }
+
+ # Check magic bytes
+ valid = False
+ ext_lower = ext.lower()
+ magic_list = audio_magic_bytes.get(ext_lower, [])
+ for magic in magic_list:
+ if len(audio_bytes) >= len(magic) and audio_bytes.startswith(magic):
+ valid = True
+ break
+
+ if not valid:
+ raise ValueError(
+ f"Invalid audio file: {file_path}. File signature does not match expected format {ext_lower}"
+ )
+
+ # Extract audio metadata (placeholder)
+ duration = 0
+ sample_rate = 0
+ channels = 0
+ format_str = ext[1:].upper()
+
+ # Create ResourceNode - metadata only, no content understanding yet
+ root_node = ResourceNode(
+ type=NodeType.ROOT,
+ title=file_path.stem,
+ level=0,
+ detail_file=None,
+ content_path=None,
+ children=[],
+ meta={
+ "duration": duration,
+ "sample_rate": sample_rate,
+ "channels": channels,
+ "format": format_str.lower(),
+ "content_type": "audio",
+ "source_title": file_path.stem,
+ "semantic_name": file_path.stem,
+ "original_filename": original_filename,
+ },
+ )
+
+ # Phase 3: Build directory structure (handled by TreeBuilder)
+ return ParseResult(
+ root=root_node,
+ source_path=str(file_path),
+ temp_dir_path=temp_uri,
+ source_format="audio",
+ parser_name="AudioParser",
+ meta={"content_type": "audio", "format": format_str.lower()},
+ )
+
+ async def _asr_transcribe(self, audio_bytes: bytes, model: Optional[str]) -> str:
+ """
+ Generate audio transcription using ASR.
+
+ Args:
+ audio_bytes: Audio binary data
+ model: ASR model name
+
+ Returns:
+ Audio transcription in markdown format
+
+ TODO: Integrate with actual ASR API (Whisper, etc.)
+ """
+ # Fallback implementation - returns basic placeholder
+ return "Audio transcription (ASR integration pending)\n\nThis is an audio. ASR transcription feature has not yet integrated external API."
+
+ async def _asr_transcribe_with_timestamps(
+ self, audio_bytes: bytes, model: Optional[str]
+ ) -> Optional[str]:
+ """
+ Extract transcription with timestamps from audio using ASR.
+
+ Args:
+ audio_bytes: Audio binary data
+ model: ASR model name
+
+ Returns:
+ Transcript with timestamps in markdown format, or None if not available
+
+ TODO: Integrate with ASR API
+ """
+ # Not implemented - return None
+ return None
+
+ async def _generate_semantic_info(
+ self, node: ResourceNode, description: str, viking_fs, has_transcript: bool
+ ):
+ """
+ Phase 2: Generate abstract and overview.
+
+ Args:
+ node: ResourceNode to update
+ description: Audio description
+ viking_fs: VikingFS instance
+ has_transcript: Whether transcript file exists
+ """
+ # Generate abstract (short summary, < 100 tokens)
+ abstract = description[:200] if len(description) > 200 else description
+
+ # Generate overview (content summary + file list + usage instructions)
+ overview_parts = [
+ "## Content Summary\n",
+ description,
+ "\n\n## Available Files\n",
+ f"- {node.meta['original_filename']}: Original audio file ({node.meta['duration']}s, {node.meta['sample_rate']}Hz, {node.meta['channels']}ch, {node.meta['format'].upper()} format)\n",
+ ]
+
+ if has_transcript:
+ overview_parts.append("- transcript.md: Transcript with timestamps from the audio\n")
+
+ overview_parts.append("\n## Usage\n")
+ overview_parts.append("### Play Audio\n")
+ overview_parts.append("```python\n")
+ overview_parts.append("audio_bytes = await audio_resource.play()\n")
+ overview_parts.append("# Returns: Audio file binary data\n")
+ overview_parts.append("# Purpose: Play or save the audio\n")
+ overview_parts.append("```\n\n")
+
+ if has_transcript:
+ overview_parts.append("### Get Timestamps Transcript\n")
+ overview_parts.append("```python\n")
+ overview_parts.append("timestamps = await audio_resource.timestamps()\n")
+ overview_parts.append("# Returns: FileContent object or None\n")
+ overview_parts.append("# Purpose: Extract timestamped transcript from the audio\n")
+ overview_parts.append("```\n\n")
+
+ overview_parts.append("### Get Audio Metadata\n")
+ overview_parts.append("```python\n")
+ overview_parts.append(
+ f"duration = audio_resource.get_duration() # {node.meta['duration']}s\n"
+ )
+ overview_parts.append(
+ f"sample_rate = audio_resource.get_sample_rate() # {node.meta['sample_rate']}Hz\n"
+ )
+ overview_parts.append(
+ f"channels = audio_resource.get_channels() # {node.meta['channels']}\n"
+ )
+ overview_parts.append(f'format = audio_resource.get_format() # "{node.meta["format"]}"\n')
+ overview_parts.append("```\n")
+
+ overview = "".join(overview_parts)
+
+ # Store in node meta
+ node.meta["abstract"] = abstract
+ node.meta["overview"] = overview
+
+ async def parse_content(
+ self, content: str, source_path: Optional[str] = None, instruction: str = "", **kwargs
+ ) -> ParseResult:
+ """
+ Parse audio from content string - Not yet implemented.
+
+ Args:
+ content: Audio content (base64 or binary string)
+ source_path: Optional source path for metadata
+ **kwargs: Additional parsing parameters
+
+ Returns:
+ ParseResult with audio content
+
+ Raises:
+ NotImplementedError: This feature is not yet implemented
+ """
+ raise NotImplementedError("Audio parsing from content not yet implemented")
diff --git a/openviking/parse/parsers/media/constants.py b/openviking/parse/parsers/media/constants.py
new file mode 100644
index 000000000..f8bc96cf2
--- /dev/null
+++ b/openviking/parse/parsers/media/constants.py
@@ -0,0 +1,15 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+"""Constants for media parsers."""
+
+# Image extensions supported by ImageParser
+IMAGE_EXTENSIONS = [".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp", ".svg"]
+
+# Audio extensions supported by AudioParser
+AUDIO_EXTENSIONS = [".mp3", ".wav", ".ogg", ".flac", ".aac", ".m4a", ".opus"]
+
+# Video extensions supported by VideoParser
+VIDEO_EXTENSIONS = [".mp4", ".avi", ".mov", ".mkv", ".webm", ".flv", ".wmv"]
+
+# All media extensions combined
+MEDIA_EXTENSIONS = set(IMAGE_EXTENSIONS + AUDIO_EXTENSIONS + VIDEO_EXTENSIONS)
diff --git a/openviking/parse/parsers/media/image.py b/openviking/parse/parsers/media/image.py
new file mode 100644
index 000000000..3442f26e8
--- /dev/null
+++ b/openviking/parse/parsers/media/image.py
@@ -0,0 +1,286 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+"""
+Media parser interfaces for OpenViking - Future expansion.
+
+This module defines parser interfaces for media types (image, audio, video).
+These are placeholder implementations that raise NotImplementedError.
+They serve as a design reference for future media parsing capabilities.
+
+For current document parsing (PDF, Markdown, HTML, Text), see other parser modules.
+"""
+
+from pathlib import Path
+from typing import List, Optional, Union
+
+from PIL import Image
+
+from openviking.parse.base import NodeType, ParseResult, ResourceNode
+from openviking.parse.parsers.base_parser import BaseParser
+from openviking.parse.parsers.media.constants import IMAGE_EXTENSIONS
+from openviking.prompts import render_prompt
+from openviking.storage.viking_fs import get_viking_fs
+from openviking_cli.utils.config import get_openviking_config
+from openviking_cli.utils.config.parser_config import ImageConfig
+from openviking_cli.utils.logger import get_logger
+from openviking_cli.utils.uri import VikingURI
+
+logger = get_logger(__name__)
+
+# =============================================================================
+# Configuration Classes
+# =============================================================================
+
+
+# =============================================================================
+# Parser Classes
+# =============================================================================
+
+
+class ImageParser(BaseParser):
+ """
+ Image parser - Future implementation.
+
+ Planned Features:
+ 1. Visual content understanding using VLM (Vision Language Model)
+ 2. OCR text extraction for images containing text
+ 3. Metadata extraction (dimensions, format, EXIF data)
+ 4. Generate semantic description and structured ResourceNode
+
+ Example workflow:
+ 1. Load image file
+ 2. (Optional) Perform OCR to extract text
+ 3. (Optional) Use VLM to generate visual description
+ 4. Create ResourceNode with image metadata and descriptions
+ 5. Return ParseResult
+
+ Supported formats: PNG, JPG, JPEG, GIF, BMP, WEBP, SVG
+ """
+
+ def __init__(self, config: Optional[ImageConfig] = None, **kwargs):
+ """
+ Initialize ImageParser.
+
+ Args:
+ config: Image parsing configuration
+ **kwargs: Additional configuration parameters
+ """
+ self.config = config or ImageConfig()
+
+ @property
+ def supported_extensions(self) -> List[str]:
+ """Return supported image file extensions."""
+ return IMAGE_EXTENSIONS
+
+ async def parse(self, source: Union[str, Path], instruction: str = "", **kwargs) -> ParseResult:
+ """
+ Parse image file - only copy original file and extract basic metadata, no content understanding.
+
+ Args:
+ source: Image file path
+ **kwargs: Additional parsing parameters
+
+ Returns:
+ ParseResult with image content
+
+ Raises:
+ FileNotFoundError: If source file does not exist
+ IOError: If image processing fails
+ """
+ # Convert to Path object
+ file_path = Path(source) if isinstance(source, str) else source
+ if not file_path.exists():
+ raise FileNotFoundError(f"Image file not found: {source}")
+
+ viking_fs = get_viking_fs()
+ temp_uri = viking_fs.create_temp_uri()
+
+ # Phase 1: Generate temporary files
+ image_bytes = file_path.read_bytes()
+ ext = file_path.suffix
+
+ # Sanitize original filename (replace spaces with underscores)
+ original_filename = file_path.name.replace(" ", "_")
+ # Root directory name: filename stem + _ + extension (without dot)
+ stem = file_path.stem.replace(" ", "_")
+ ext_no_dot = ext[1:] if ext else ""
+ root_dir_name = VikingURI.sanitize_segment(f"{stem}_{ext_no_dot}")
+ root_dir_uri = f"{temp_uri}/{root_dir_name}"
+ await viking_fs.mkdir(root_dir_uri)
+
+ # 1.1 Save original image with original filename (sanitized)
+ await viking_fs.write_file_bytes(f"{root_dir_uri}/{original_filename}", image_bytes)
+
+ # 1.2 Validate and extract image metadata
+ try:
+ img = Image.open(file_path)
+ img.verify() # Verify that it's a valid image
+ img.close() # Close and reopen to reset after verify()
+ img = Image.open(file_path)
+ width, height = img.size
+ format_str = img.format or ext[1:].upper()
+ except Exception as e:
+ raise ValueError(f"Invalid image file: {file_path}. Error: {e}") from e
+
+ # Create ResourceNode - metadata only, no content understanding yet
+ root_node = ResourceNode(
+ type=NodeType.ROOT,
+ title=file_path.stem,
+ level=0,
+ detail_file=None,
+ content_path=None,
+ children=[],
+ meta={
+ "width": width,
+ "height": height,
+ "format": format_str.lower(),
+ "content_type": "image",
+ "source_title": file_path.stem,
+ "semantic_name": file_path.stem,
+ "original_filename": original_filename,
+ },
+ )
+
+ # Phase 3: Build directory structure (handled by TreeBuilder)
+ return ParseResult(
+ root=root_node,
+ source_path=str(file_path),
+ temp_dir_path=temp_uri,
+ source_format="image",
+ parser_name="ImageParser",
+ meta={"content_type": "image", "format": format_str.lower()},
+ )
+
+ async def _vlm_describe(self, image_bytes: bytes, model: Optional[str]) -> str:
+ """
+ Generate image description using VLM.
+
+ Args:
+ image_bytes: Image binary data
+ model: VLM model name
+
+ Returns:
+ Image description in markdown format
+ """
+ try:
+ vlm = get_openviking_config().vlm
+
+ # Render prompt
+ prompt = render_prompt(
+ "parsing.image_summary",
+ {
+ "context": "No additional context",
+ },
+ )
+ response = await vlm.get_vision_completion_async(
+ prompt=prompt,
+ images=[image_bytes],
+ )
+ logger.info(
+ f"[ImageParser._vlm_describe] VLM response received, length: {len(response)}, content: {response[:256]}"
+ )
+
+ return response.strip()
+
+ except Exception as e:
+ logger.error(
+ f"[ImageParser._vlm_describe] Error in VLM image description: {e}", exc_info=True
+ )
+ # Fallback to basic description
+ return "Image description (VLM integration failed)\n\nThis is an image file."
+
+ async def _ocr_extract(self, image_bytes: bytes, lang: str) -> Optional[str]:
+ """
+ Extract text from image using OCR.
+
+ Args:
+ image_bytes: Image binary data
+ lang: OCR language code
+
+ Returns:
+ Extracted text in markdown format, or None if no text found
+
+ TODO: Integrate with OCR API (Tesseract, PaddleOCR, etc.)
+ """
+ # Not implemented - return None
+ return None
+
+ async def _generate_semantic_info(
+ self, node: ResourceNode, description: str, viking_fs, has_ocr: bool, root_dir_uri: str
+ ):
+ """
+ Phase 2: Generate abstract and overview and write to .abstract.md and .overview.md.
+
+ Args:
+ node: ResourceNode to update
+ description: Image description
+ viking_fs: VikingFS instance
+ has_ocr: Whether OCR file exists
+ root_dir_uri: Root directory URI to write semantic files
+ """
+ # Generate abstract (short summary, < 100 tokens)
+ abstract = description[:253] + "..." if len(description) > 256 else description
+
+ # Generate overview (content summary + file list + usage instructions)
+ overview_parts = [
+ "## Content Summary\n",
+ description,
+ "\n\n## Available Files\n",
+ f"- {node.meta['original_filename']}: Original image file ({node.meta['width']}x{node.meta['height']}, {node.meta['format'].upper()} format)\n",
+ ]
+
+ if has_ocr:
+ overview_parts.append("- ocr.md: OCR text recognition result from the image\n")
+
+ overview_parts.append("\n## Usage\n")
+ overview_parts.append("### View Image\n")
+ overview_parts.append("```python\n")
+ overview_parts.append("image_bytes = await image_resource.view()\n")
+ overview_parts.append("# Returns: PNG/JPG format image binary data\n")
+ overview_parts.append("# Purpose: Display or save the image\n")
+ overview_parts.append("```\n\n")
+
+ if has_ocr:
+ overview_parts.append("### Get OCR-recognized Text\n")
+ overview_parts.append("```python\n")
+ overview_parts.append("ocr_text = await image_resource.ocr()\n")
+ overview_parts.append("# Returns: FileContent object or None\n")
+ overview_parts.append("# Purpose: Extract text information from the image\n")
+ overview_parts.append("```\n\n")
+
+ overview_parts.append("### Get Image Metadata\n")
+ overview_parts.append("```python\n")
+ overview_parts.append(
+ f"size = image_resource.get_size() # ({node.meta['width']}, {node.meta['height']})\n"
+ )
+ overview_parts.append(f'format = image_resource.get_format() # "{node.meta["format"]}"\n')
+ overview_parts.append("```\n")
+
+ overview = "".join(overview_parts)
+
+ # Store in node meta
+ node.meta["abstract"] = abstract
+ node.meta["overview"] = overview
+
+ # Write to files in temp directory
+ # await viking_fs.write_file(f"{root_dir_uri}/.abstract.md", abstract)
+ # await viking_fs.write_file(f"{root_dir_uri}/.overview.md", overview)
+
+ async def parse_content(
+ self, content: str, source_path: Optional[str] = None, instruction: str = "", **kwargs
+ ) -> ParseResult:
+ """
+ Parse image from content string - Not yet implemented.
+
+ Args:
+ content: Image content (base64 or binary string)
+ source_path: Optional source path for metadata
+ **kwargs: Additional parsing parameters
+
+ Returns:
+ ParseResult with image content
+
+ Raises:
+ NotImplementedError: This feature is not yet implemented
+ """
+ raise NotImplementedError("Image parsing not yet implemented")
diff --git a/openviking/parse/parsers/media/utils.py b/openviking/parse/parsers/media/utils.py
new file mode 100644
index 000000000..1e8ad30d5
--- /dev/null
+++ b/openviking/parse/parsers/media/utils.py
@@ -0,0 +1,153 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+"""Media-related utilities for OpenViking."""
+
+import asyncio
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from openviking.prompts import render_prompt
+from openviking.storage.viking_fs import get_viking_fs
+from openviking_cli.utils.config import get_openviking_config
+from openviking_cli.utils.logger import get_logger
+
+from .constants import AUDIO_EXTENSIONS, IMAGE_EXTENSIONS, VIDEO_EXTENSIONS
+
+logger = get_logger(__name__)
+
+
+def get_media_type(source_path: Optional[str], source_format: Optional[str]) -> Optional[str]:
+ """
+ Determine media type from source path or format.
+
+ Args:
+ source_path: Source file path
+ source_format: Source format string (e.g., "image", "audio", "video")
+
+ Returns:
+ Media type ("image", "audio", "video") or None if not a media file
+ """
+ if source_format:
+ if source_format in ["image", "audio", "video"]:
+ return source_format
+
+ if source_path:
+ ext = Path(source_path).suffix.lower()
+ if ext in IMAGE_EXTENSIONS:
+ return "image"
+ elif ext in AUDIO_EXTENSIONS:
+ return "audio"
+ elif ext in VIDEO_EXTENSIONS:
+ return "video"
+
+ return None
+
+
+def get_media_base_uri(media_type: str) -> str:
+ """
+ Get base URI for media files.
+
+ Args:
+ media_type: Media type ("image", "audio", "video")
+
+ Returns:
+ Base URI like "viking://resources/images/20250219"
+ """
+ # Map singular media types to plural directory names
+ media_dir_map = {"image": "images", "audio": "audio", "video": "video"}
+ media_dir = media_dir_map.get(media_type, media_type)
+ # Get current date in YYYYMMDD format
+ date_str = datetime.now().strftime("%Y%m%d")
+ return f"viking://resources/{media_dir}/{date_str}"
+
+
+async def generate_image_summary(
+ image_uri: str, original_filename: str, llm_sem: Optional[asyncio.Semaphore] = None
+) -> Dict[str, Any]:
+ """
+ Generate summary for an image file using VLM.
+
+ Args:
+ image_uri: URI to the image file in VikingFS
+ original_filename: Original filename of the image
+
+ Returns:
+ Dictionary with "name" and "summary" keys
+ """
+ viking_fs = get_viking_fs()
+ vlm = get_openviking_config().vlm
+ file_name = original_filename
+
+ try:
+ # Read image bytes
+ image_bytes = await viking_fs.read_file_bytes(image_uri)
+ if not isinstance(image_bytes, bytes):
+ raise ValueError(f"Expected bytes for image file, got {type(image_bytes)}")
+
+ logger.info(
+ f"[MediaUtils.generate_image_summary] Generating summary for image: {image_uri}"
+ )
+
+ # Render prompt
+ prompt = render_prompt(
+ "parsing.image_summary",
+ {"context": "No additional context"},
+ )
+
+ # Call VLM
+ async with llm_sem or asyncio.Semaphore(1):
+ response = await vlm.get_vision_completion_async(
+ prompt=prompt,
+ images=[image_bytes],
+ )
+
+ logger.info(
+ f"[MediaUtils.generate_image_summary] VLM response received, length: {len(response)}"
+ )
+ return {"name": file_name, "summary": response.strip()}
+
+ except Exception as e:
+ logger.error(
+ f"[MediaUtils.generate_image_summary] Failed to generate image summary: {e}",
+ exc_info=True,
+ )
+ return {"name": file_name, "summary": "Image summary generation failed"}
+
+
+async def generate_audio_summary(
+ audio_uri: str, original_filename: str, llm_sem: Optional[asyncio.Semaphore] = None
+) -> Dict[str, Any]:
+ """
+ Generate summary for an audio file (placeholder).
+
+ Args:
+ audio_uri: URI to the audio file in VikingFS
+ original_filename: Original filename of the audio
+
+ Returns:
+ Dictionary with "name" and "summary" keys
+ """
+ logger.info(
+ f"[MediaUtils.generate_audio_summary] Audio summary generation not yet implemented for: {audio_uri}"
+ )
+ return {"name": original_filename, "summary": "Audio summary generation not yet implemented"}
+
+
+async def generate_video_summary(
+ video_uri: str, original_filename: str, llm_sem: Optional[asyncio.Semaphore] = None
+) -> Dict[str, Any]:
+ """
+ Generate summary for a video file (placeholder).
+
+ Args:
+ video_uri: URI to the video file in VikingFS
+ original_filename: Original filename of the video
+
+ Returns:
+ Dictionary with "name" and "summary" keys
+ """
+ logger.info(
+ f"[MediaUtils.generate_video_summary] Video summary generation not yet implemented for: {video_uri}"
+ )
+ return {"name": original_filename, "summary": "Video summary generation not yet implemented"}
diff --git a/openviking/parse/parsers/media/video.py b/openviking/parse/parsers/media/video.py
new file mode 100644
index 000000000..a0a07602f
--- /dev/null
+++ b/openviking/parse/parsers/media/video.py
@@ -0,0 +1,256 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+"""
+Video parser - Future implementation.
+
+Planned Features:
+1. Key frame extraction at regular intervals
+2. Audio track transcription using ASR
+3. VLM-based scene description for key frames
+4. Video metadata extraction (duration, resolution, codec)
+5. Generate structured ResourceNode combining visual and audio
+
+Example workflow:
+ 1. Load video file
+ 2. Extract metadata (duration, resolution, fps)
+ 3. Extract audio track → transcribe using AudioParser
+ 4. Extract key frames at specified intervals
+ 5. For each frame: generate VLM description
+ 6. Create ResourceNode tree:
+ - Root: video metadata
+ - Children: timeline nodes (each with frame + transcript)
+ 7. Return ParseResult
+
+Supported formats: MP4, AVI, MOV, MKV, WEBM
+"""
+
+from pathlib import Path
+from typing import List, Optional, Union
+
+from openviking.parse.base import NodeType, ParseResult, ResourceNode
+from openviking.parse.parsers.base_parser import BaseParser
+from openviking.parse.parsers.media.constants import VIDEO_EXTENSIONS
+from openviking_cli.utils.config.parser_config import VideoConfig
+
+
+class VideoParser(BaseParser):
+ """
+ Video parser for video files.
+ """
+
+ def __init__(self, config: Optional[VideoConfig] = None, **kwargs):
+ """
+ Initialize VideoParser.
+
+ Args:
+ config: Video parsing configuration
+ **kwargs: Additional configuration parameters
+ """
+ self.config = config or VideoConfig()
+
+ @property
+ def supported_extensions(self) -> List[str]:
+ """Return supported video file extensions."""
+ return VIDEO_EXTENSIONS
+
+ async def parse(self, source: Union[str, Path], instruction: str = "", **kwargs) -> ParseResult:
+ """
+ Parse video file - only copy original file and extract basic metadata, no content understanding.
+
+ Args:
+ source: Video file path
+ **kwargs: Additional parsing parameters
+
+ Returns:
+ ParseResult with video content
+
+ Raises:
+ FileNotFoundError: If source file does not exist
+ IOError: If video processing fails
+ """
+ from openviking.storage.viking_fs import get_viking_fs
+
+ # Convert to Path object
+ file_path = Path(source) if isinstance(source, str) else source
+ if not file_path.exists():
+ raise FileNotFoundError(f"Video file not found: {source}")
+
+ viking_fs = get_viking_fs()
+ temp_uri = viking_fs.create_temp_uri()
+
+ # Phase 1: Generate temporary files
+ video_bytes = file_path.read_bytes()
+ ext = file_path.suffix
+
+ from openviking_cli.utils.uri import VikingURI
+
+ # Sanitize original filename (replace spaces with underscores)
+ original_filename = file_path.name.replace(" ", "_")
+ # Root directory name: filename stem + _ + extension (without dot)
+ stem = file_path.stem.replace(" ", "_")
+ ext_no_dot = ext[1:] if ext else ""
+ root_dir_name = VikingURI.sanitize_segment(f"{stem}_{ext_no_dot}")
+ root_dir_uri = f"{temp_uri}/{root_dir_name}"
+ await viking_fs.mkdir(root_dir_uri)
+
+ # 1.1 Save original video with original filename (sanitized)
+ await viking_fs.write_file_bytes(f"{root_dir_uri}/{original_filename}", video_bytes)
+
+ # 1.2 Validate video file using magic bytes
+ # Define magic bytes for supported video formats
+ video_magic_bytes = {
+ ".mp4": [b"\x00\x00\x00", b"ftyp"],
+ ".avi": [b"RIFF"],
+ ".mov": [b"\x00\x00\x00", b"ftyp"],
+ ".mkv": [b"\x1a\x45\xdf\xa3"],
+ ".webm": [b"\x1a\x45\xdf\xa3"],
+ ".flv": [b"FLV"],
+ ".wmv": [b"\x30\x26\xb2\x75\x8e\x66\xcf\x11"],
+ }
+
+ # Check magic bytes
+ valid = False
+ ext_lower = ext.lower()
+ magic_list = video_magic_bytes.get(ext_lower, [])
+ for magic in magic_list:
+ if len(video_bytes) >= len(magic) and video_bytes.startswith(magic):
+ valid = True
+ break
+
+ if not valid:
+ raise ValueError(
+ f"Invalid video file: {file_path}. File signature does not match expected format {ext_lower}"
+ )
+
+ # Extract video metadata (placeholder)
+ duration = 0
+ width = 0
+ height = 0
+ fps = 0
+ format_str = ext[1:].upper()
+
+ # Create ResourceNode - metadata only, no content understanding yet
+ root_node = ResourceNode(
+ type=NodeType.ROOT,
+ title=file_path.stem,
+ level=0,
+ detail_file=None,
+ content_path=None,
+ children=[],
+ meta={
+ "duration": duration,
+ "width": width,
+ "height": height,
+ "fps": fps,
+ "format": format_str.lower(),
+ "content_type": "video",
+ "source_title": file_path.stem,
+ "semantic_name": file_path.stem,
+ "original_filename": original_filename,
+ },
+ )
+
+ # Phase 3: Build directory structure (handled by TreeBuilder)
+ return ParseResult(
+ root=root_node,
+ source_path=str(file_path),
+ temp_dir_path=temp_uri,
+ source_format="video",
+ parser_name="VideoParser",
+ meta={"content_type": "video", "format": format_str.lower()},
+ )
+
+ async def _generate_video_description(self, file_path: Path, config: VideoConfig) -> str:
+ """
+ Generate video description using key frames and audio transcription.
+
+ Args:
+ file_path: Video file path
+ config: Video parsing configuration
+
+ Returns:
+ Video description in markdown format
+
+ TODO: Integrate with actual video processing libraries
+ """
+ # Fallback implementation - returns basic placeholder
+ return "Video description (video processing integration pending)\n\nThis is a video. Video processing feature has not yet integrated external libraries."
+
+ async def _generate_semantic_info(
+ self, node: ResourceNode, description: str, viking_fs, has_key_frames: bool
+ ):
+ """
+ Phase 2: Generate abstract and overview.
+
+ Args:
+ node: ResourceNode to update
+ description: Video description
+ viking_fs: VikingFS instance
+ has_key_frames: Whether key frames directory exists
+ """
+ # Generate abstract (short summary, < 100 tokens)
+ abstract = description[:200] if len(description) > 200 else description
+
+ # Generate overview (content summary + file list + usage instructions)
+ overview_parts = [
+ "## Content Summary\n",
+ description,
+ "\n\n## Available Files\n",
+ f"- {node.meta['original_filename']}: Original video file ({node.meta['duration']}s, {node.meta['width']}x{node.meta['height']}, {node.meta['fps']}fps, {node.meta['format'].upper()} format)\n",
+ ]
+
+ if has_key_frames:
+ overview_parts.append("- keyframes/: Directory containing extracted key frames\n")
+
+ overview_parts.append("\n## Usage\n")
+ overview_parts.append("### Play Video\n")
+ overview_parts.append("```python\n")
+ overview_parts.append("video_bytes = await video_resource.play()\n")
+ overview_parts.append("# Returns: Video file binary data\n")
+ overview_parts.append("# Purpose: Play or save the video\n")
+ overview_parts.append("```\n\n")
+
+ if has_key_frames:
+ overview_parts.append("### Get Key Frames\n")
+ overview_parts.append("```python\n")
+ overview_parts.append("keyframes = await video_resource.keyframes()\n")
+ overview_parts.append("# Returns: List of key frame resources\n")
+ overview_parts.append("# Purpose: Analyze video scenes\n")
+ overview_parts.append("```\n\n")
+
+ overview_parts.append("### Get Video Metadata\n")
+ overview_parts.append("```python\n")
+ overview_parts.append(
+ f"duration = video_resource.get_duration() # {node.meta['duration']}s\n"
+ )
+ overview_parts.append(
+ f"resolution = video_resource.get_resolution() # ({node.meta['width']}, {node.meta['height']})\n"
+ )
+ overview_parts.append(f"fps = video_resource.get_fps() # {node.meta['fps']}\n")
+ overview_parts.append(f'format = video_resource.get_format() # "{node.meta["format"]}"\n')
+ overview_parts.append("```\n")
+
+ overview = "".join(overview_parts)
+
+ # Store in node meta
+ node.meta["abstract"] = abstract
+ node.meta["overview"] = overview
+
+ async def parse_content(
+ self, content: str, source_path: Optional[str] = None, instruction: str = "", **kwargs
+ ) -> ParseResult:
+ """
+ Parse video from content string - Not yet implemented.
+
+ Args:
+ content: Video content (base64 or binary string)
+ source_path: Optional source path for metadata
+ **kwargs: Additional parsing parameters
+
+ Returns:
+ ParseResult with video content
+
+ Raises:
+ NotImplementedError: This feature is not yet implemented
+ """
+ raise NotImplementedError("Video parsing from content not yet implemented")
diff --git a/openviking/parse/registry.py b/openviking/parse/registry.py
index 526b2272e..5777edecb 100644
--- a/openviking/parse/registry.py
+++ b/openviking/parse/registry.py
@@ -12,12 +12,15 @@ from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Union
from openviking.parse.base import ParseResult
from openviking.parse.parsers.base_parser import BaseParser
+from openviking.parse.parsers.code import CodeRepositoryParser
+from openviking.parse.parsers.directory import DirectoryParser
from openviking.parse.parsers.epub import EPubParser
from openviking.parse.parsers.excel import ExcelParser
# Import will be handled dynamically to avoid dependency issues
from openviking.parse.parsers.html import HTMLParser
from openviking.parse.parsers.markdown import MarkdownParser
+from openviking.parse.parsers.media import AudioParser, ImageParser, VideoParser
from openviking.parse.parsers.pdf import PDFParser
from openviking.parse.parsers.powerpoint import PowerPointParser
from openviking.parse.parsers.text import TextParser
@@ -34,7 +37,7 @@ logger = logging.getLogger(__name__)
class ParserRegistry:
"""
- Registry for document parsers.
+ Registry for document parsers, which is a singleton.
Automatically selects appropriate parser based on file extension.
"""
@@ -46,6 +49,7 @@ class ParserRegistry:
Args:
register_optional: Whether to register optional parsers
that require extra dependencies
+ parser_configs: Dictionary of parser configurations (from load_parser_configs_from_dict)
"""
self._parsers: Dict[str, BaseParser] = {}
self._extension_map: Dict[str, str] = {}
@@ -54,7 +58,7 @@ class ParserRegistry:
self.register("text", TextParser())
self.register("markdown", MarkdownParser())
self.register("pdf", PDFParser())
- self.register("html", HTMLParser())
+ self.register("html", HTMLParser()) # HTMLParser doesn't accept config yet
# Register markitdown-inspired parsers (built-in)
self.register("word", WordParser())
@@ -62,32 +66,12 @@ class ParserRegistry:
self.register("excel", ExcelParser())
self.register("epub", EPubParser())
self.register("zip", ZipParser())
+ self.register("code", CodeRepositoryParser())
+ self.register("directory", DirectoryParser())
- # Register code parser dynamically
- try:
- from openviking.parse.parsers.code import CodeRepositoryParser
-
- self.register("code", CodeRepositoryParser())
- except ImportError as e:
- logger.warning(f"CodeRepositoryParser not available: {e}")
-
- # Register directory parser
- try:
- from openviking.parse.parsers.directory import DirectoryParser
-
- self.register("directory", DirectoryParser())
- except ImportError as e:
- logger.warning(f"DirectoryParser not available: {e}")
-
- # Register optional media parsers
- if register_optional:
- try:
- from openviking.parse.parsers.media import ImageParser
-
- self.register("image", ImageParser())
- logger.info("Registered ImageParser for image formats")
- except ImportError as e:
- logger.debug(f"ImageParser not registered: {e}")
+ self.register("image", ImageParser())
+ self.register("audio", AudioParser())
+ self.register("video", VideoParser())
def register(self, name: str, parser: BaseParser) -> None:
"""
diff --git a/openviking/parse/tree_builder.py b/openviking/parse/tree_builder.py
index 903e822d2..7d3f5a891 100644
--- a/openviking/parse/tree_builder.py
+++ b/openviking/parse/tree_builder.py
@@ -24,6 +24,8 @@ import logging
from typing import TYPE_CHECKING, Optional
from openviking.core.building_tree import BuildingTree
+from openviking.parse.parsers.media.utils import get_media_base_uri, get_media_type
+from openviking.storage.queuefs import SemanticMsg, get_queue_manager
from openviking.storage.viking_fs import get_viking_fs
from openviking_cli.utils.uri import VikingURI
@@ -58,10 +60,15 @@ class TreeBuilder:
"""Initialize TreeBuilder."""
pass
- def _get_base_uri(self, scope: str) -> str:
- """Get base URI for scope."""
- # Resources are now in independent resources scope
+ def _get_base_uri(
+ self, scope: str, source_path: Optional[str] = None, source_format: Optional[str] = None
+ ) -> str:
+ """Get base URI for scope, with special handling for media files."""
+ # Check if it's a media file first
if scope == "resources":
+ media_type = get_media_type(source_path, source_format)
+ if media_type:
+ return get_media_base_uri(media_type)
return "viking://resources"
if scope == "user":
# user resources go to memories (no separate resources dir)
@@ -93,6 +100,7 @@ class TreeBuilder:
temp_dir_path: Temporary directory Viking URI (e.g., viking://temp/xxx)
scope: Scope ("resources", "user", or "agent")
base_uri: Base URI (None = use scope default)
+ source_node: Source ResourceNode
source_path: Source file path
source_format: Source file format
@@ -108,41 +116,45 @@ class TreeBuilder:
doc_dirs = [e for e in entries if e.get("isDir") and e["name"] not in [".", ".."]]
if len(doc_dirs) != 1:
- raise ValueError(f"Expected 1 document directory in {temp_uri}, found {len(doc_dirs)}")
+ logger.error(
+ f"[TreeBuilder] Expected 1 document directory in {temp_uri}, found {len(doc_dirs)}"
+ )
+ raise ValueError(
+ f"[TreeBuilder] Expected 1 document directory in {temp_uri}, found {len(doc_dirs)}"
+ )
- doc_name = doc_dirs[0]["name"]
- doc_uri = f"{temp_uri}/{doc_name}"
+ doc_name = VikingURI.sanitize_segment(doc_dirs[0]["name"])
+ temp_doc_uri = f"{temp_uri}/{doc_name}"
# 2. Determine base_uri
if base_uri is None:
- base_uri = self._get_base_uri(scope)
-
- logger.info(f"Finalizing from temp: {temp_uri} -> {base_uri}")
+ base_uri = self._get_base_uri(scope, source_path, source_format)
# 3. Build final URI, auto-renaming on conflict (e.g. doc_1, doc_2, ...)
candidate_uri = VikingURI(base_uri).join(doc_name).uri
final_uri = await self._resolve_unique_uri(candidate_uri)
if final_uri != candidate_uri:
- logger.info(f"Resolved name conflict: {candidate_uri} -> {final_uri}")
+ logger.info(f"[TreeBuilder] Resolved name conflict: {candidate_uri} -> {final_uri}")
+ else:
+ logger.info(f"[TreeBuilder] Finalizing from temp: {final_uri}")
# 4. Move directory tree from temp to final location in AGFS
- await self._move_directory_in_agfs(doc_uri, final_uri)
- logger.info(f"Moved temp tree: {doc_uri} -> {final_uri}")
+ await self._move_directory_in_agfs(temp_doc_uri, final_uri)
+ logger.info(f"[TreeBuilder] Moved temp tree: {temp_doc_uri} -> {final_uri}")
# 5. Cleanup temporary root directory
try:
await viking_fs.delete_temp(temp_uri)
- logger.info(f"Cleaned up temp root: {temp_uri}")
+ logger.info(f"[TreeBuilder] Cleaned up temp root: {temp_uri}")
except Exception as e:
- logger.warning(f"Failed to cleanup temp root: {e}")
+ logger.warning(f"[TreeBuilder] Failed to cleanup temp root: {e}")
# 6. Enqueue to SemanticQueue for async semantic generation
try:
- context_type = "resource" # Default to resource
- await self._enqueue_semantic_generation(final_uri, context_type)
- logger.info(f"Enqueued semantic generation for: {final_uri}")
+ await self._enqueue_semantic_generation(final_uri, "resource")
+ logger.info(f"[TreeBuilder] Enqueued semantic generation for: {final_uri}")
except Exception as e:
- logger.error(f"Failed to enqueue semantic generation: {e}", exc_info=True)
+ logger.error(f"[TreeBuilder] Failed to enqueue semantic generation: {e}", exc_info=True)
# 7. Return simple BuildingTree (no scanning needed)
tree = BuildingTree(
@@ -151,8 +163,6 @@ class TreeBuilder:
)
tree._root_uri = final_uri
- logger.info(f"Finalized tree: root_uri={final_uri}")
-
return tree
async def _resolve_unique_uri(self, uri: str, max_attempts: int = 100) -> str:
@@ -179,9 +189,7 @@ class TreeBuilder:
if not await _exists(candidate):
return candidate
- raise FileExistsError(
- f"Cannot resolve unique name for {uri} after {max_attempts} attempts"
- )
+ raise FileExistsError(f"Cannot resolve unique name for {uri} after {max_attempts} attempts")
async def _move_directory_in_agfs(self, src_uri: str, dst_uri: str) -> None:
"""Recursively move AGFS directory tree (copy + delete)."""
@@ -244,7 +252,6 @@ class TreeBuilder:
uri: Directory URI to enqueue
context_type: resource/memory/skill
"""
- from openviking.storage.queuefs import SemanticMsg, get_queue_manager
queue_manager = get_queue_manager()
diff --git a/openviking/prompts/templates/parsing/image_summary.yaml b/openviking/prompts/templates/parsing/image_summary.yaml
new file mode 100644
index 000000000..26bf3f0fe
--- /dev/null
+++ b/openviking/prompts/templates/parsing/image_summary.yaml
@@ -0,0 +1,31 @@
+metadata:
+ id: "parsing.image_summary"
+ name: "Image Summary"
+ description: "Generate concise image summary for semantic parsing"
+ version: "1.0.0"
+ language: "en"
+ category: "parsing"
+
+variables:
+ - name: "context"
+ type: "string"
+ description: "Additional context for image understanding"
+ default: "No additional context"
+ required: false
+
+template: |
+ Please analyze this image and generate a concise summary for semantic indexing.
+
+ Context: {{ context }}
+
+ Generate a comprehensive description that includes:
+ 1. What is in the image (main subjects, objects)
+ 2. What is happening or what the image depicts
+ 3. Any text visible in the image
+ 4. Key visual elements and their relationships
+
+ Keep the description clear and detailed, suitable for semantic search and understanding.
+
+llm_config:
+ temperature: 0.0
+ supports_vision: true
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index 38da777a2..cfad652d2 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -13,6 +13,12 @@ from openviking.parse.parsers.constants import (
FILE_TYPE_DOCUMENTATION,
FILE_TYPE_OTHER,
)
+from openviking.parse.parsers.media.utils import (
+ generate_audio_summary,
+ generate_image_summary,
+ generate_video_summary,
+ get_media_type,
+)
from openviking.prompts import render_prompt
from openviking.storage.queuefs.named_queue import DequeueHandlerBase
from openviking.storage.queuefs.semantic_dag import DagStats, SemanticDagExecutor
@@ -250,11 +256,8 @@ class SemanticProcessor(DequeueHandlerBase):
if not file_paths:
return []
- sem = asyncio.Semaphore(self.max_concurrent_llm)
-
async def generate_one_summary(file_path: str) -> Dict[str, str]:
- async with sem:
- summary = await self._generate_single_file_summary(file_path)
+ summary = await self._generate_single_file_summary(file_path)
if enqueue_files and context_type and parent_uri:
try:
await self._vectorize_single_file(
@@ -273,62 +276,74 @@ class SemanticProcessor(DequeueHandlerBase):
tasks = [generate_one_summary(fp) for fp in file_paths]
return await asyncio.gather(*tasks)
- async def _generate_single_file_summary(
- self, file_path: str, llm_sem: Optional[asyncio.Semaphore] = None
+ async def _generate_text_summary(
+ self, file_path: str, file_name: str, llm_sem: asyncio.Semaphore
) -> Dict[str, str]:
- """Generate summary for a single file.
-
- Args:
- file_path: File path
-
- Returns:
- {"name": file_name, "summary": summary_content}
- """
+ """Generate summary for a single text file (code, documentation, or other text)."""
viking_fs = get_viking_fs()
vlm = get_openviking_config().vlm
- file_name = file_path.split("/")[-1]
- try:
- # Read file content (limit length)
- content = await viking_fs.read_file(file_path)
- if isinstance(content, bytes):
+ # Read file content (limit length)
+ content = await viking_fs.read_file(file_path)
+ if isinstance(content, bytes):
+ # Try to decode with error handling for text files
+ try:
content = content.decode("utf-8")
+ except UnicodeDecodeError:
+ logger.warning(f"Failed to decode file as UTF-8, skipping: {file_path}")
+ return {"name": file_name, "summary": ""}
- # Limit content length (about 10000 tokens)
- max_chars = 30000
- if len(content) > max_chars:
- content = content[:max_chars] + "\n...(truncated)"
+ # Limit content length (about 10000 tokens)
+ max_chars = 30000
+ if len(content) > max_chars:
+ content = content[:max_chars] + "\n...(truncated)"
- # Generate summary
- if not vlm.is_available():
- logger.warning("VLM not available, using empty summary")
- return {"name": file_name, "summary": ""}
+ # Generate summary
+ if not vlm.is_available():
+ logger.warning("VLM not available, using empty summary")
+ return {"name": file_name, "summary": ""}
- # Detect file type and select appropriate prompt
- file_type = self._detect_file_type(file_name)
+ # Detect file type and select appropriate prompt
+ file_type = self._detect_file_type(file_name)
- if file_type == FILE_TYPE_CODE:
- prompt_id = "semantic.code_summary"
- elif file_type == FILE_TYPE_DOCUMENTATION:
- prompt_id = "semantic.document_summary"
- else:
- prompt_id = "semantic.file_summary"
+ if file_type == FILE_TYPE_CODE:
+ prompt_id = "semantic.code_summary"
+ elif file_type == FILE_TYPE_DOCUMENTATION:
+ prompt_id = "semantic.document_summary"
+ else:
+ prompt_id = "semantic.file_summary"
- prompt = render_prompt(
- prompt_id,
- {"file_name": file_name, "content": content},
- )
+ prompt = render_prompt(
+ prompt_id,
+ {"file_name": file_name, "content": content},
+ )
- if llm_sem:
- async with llm_sem:
- summary = await vlm.get_completion_async(prompt)
- else:
- summary = await vlm.get_completion_async(prompt)
- return {"name": file_name, "summary": summary.strip()}
+ async with llm_sem:
+ summary = await vlm.get_completion_async(prompt)
+ return {"name": file_name, "summary": summary.strip()}
- except Exception as e:
- logger.warning(f"Failed to generate summary for {file_path}: {e}")
- return {"name": file_name, "summary": ""}
+ async def _generate_single_file_summary(
+ self, file_path: str, llm_sem: Optional[asyncio.Semaphore] = None
+ ) -> Dict[str, str]:
+ """Generate summary for a single file.
+
+ Args:
+ file_path: File path
+
+ Returns:
+ {"name": file_name, "summary": summary_content}
+ """
+ file_name = file_path.split("/")[-1]
+ llm_sem = llm_sem or asyncio.Semaphore(self.max_concurrent_llm)
+ media_type = get_media_type(file_name, None)
+ if media_type == "image":
+ return await generate_image_summary(file_path, file_name, llm_sem)
+ elif media_type == "audio":
+ return await generate_audio_summary(file_path, file_name, llm_sem)
+ elif media_type == "video":
+ return await generate_video_summary(file_path, file_name, llm_sem)
+ else:
+ return await self._generate_text_summary(file_path, file_name, llm_sem)
def _extract_abstract_from_overview(self, overview_content: str) -> str:
"""Extract abstract from overview.md."""
diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.py
index 156c48dd0..ff240d984 100644
--- a/openviking/utils/resource_processor.py
+++ b/openviking/utils/resource_processor.py
@@ -119,6 +119,7 @@ class ResourceProcessor:
except Exception as e:
result["status"] = "error"
result["errors"].append(f"Parse error: {e}")
+ logger.error(f"[ResourceProcessor] Parse error: {e}")
return result
# parse_result contains:
diff --git a/openviking_cli/cli/commands/resources.py b/openviking_cli/cli/commands/resources.py
index 92940dd7e..a9bfc28fc 100644
--- a/openviking_cli/cli/commands/resources.py
+++ b/openviking_cli/cli/commands/resources.py
@@ -2,6 +2,7 @@
# SPDX-License-Identifier: Apache-2.0
"""Resource management commands."""
+from pathlib import Path
from typing import Optional
import typer
@@ -23,10 +24,62 @@ def register(app: typer.Typer) -> None:
timeout: Optional[float] = typer.Option(600.0, help="Wait timeout in seconds"),
) -> None:
"""Add resources into OpenViking."""
+ # Validate path: if it's a local path, check if it exists
+ final_path = path
+ if not (path.startswith("http://") or path.startswith("https://")):
+ unescaped_path = path.replace("\\ ", " ")
+ local_path = Path(unescaped_path)
+ final_path = unescaped_path
+ if not local_path.exists():
+ # Check if there are extra arguments (possible unquoted path with spaces)
+ import sys
+
+ # Find the index of 'add-resource' in sys.argv
+ try:
+ add_resource_idx = sys.argv.index("add-resource")
+ except ValueError:
+ add_resource_idx = sys.argv.index("add") if "add" in sys.argv else -1
+
+ if add_resource_idx != -1 and len(sys.argv) > add_resource_idx + 2:
+ # There are extra positional arguments - likely unquoted path with spaces
+ extra_args = sys.argv[add_resource_idx + 2 :]
+ suggested_path = f"{path} {' '.join(extra_args)}"
+ typer.echo(
+ typer.style(
+ f"Error: Path '{path}' does not exist.",
+ fg=typer.colors.RED,
+ bold=True,
+ ),
+ err=True,
+ )
+ typer.echo(
+ typer.style(
+ "\nIt looks like you may have forgotten to quote a path with spaces.",
+ fg=typer.colors.YELLOW,
+ ),
+ err=True,
+ )
+ typer.echo(
+ typer.style(
+ f'Suggested command: ov add-resource "{suggested_path}"',
+ fg=typer.colors.CYAN,
+ ),
+ err=True,
+ )
+ raise typer.Exit(code=1)
+ else:
+ typer.echo(
+ typer.style(
+ f"Error: Path '{path}' does not exist.", fg=typer.colors.RED, bold=True
+ ),
+ err=True,
+ )
+ raise typer.Exit(code=1)
+
run(
ctx,
lambda client: client.add_resource(
- path=path,
+ path=final_path,
target=to,
reason=reason,
instruction=instruction,
diff --git a/openviking_cli/client/http.py b/openviking_cli/client/http.py
index 395266029..a5cb69034 100644
--- a/openviking_cli/client/http.py
+++ b/openviking_cli/client/http.py
@@ -36,6 +36,7 @@ from openviking_cli.utils.config.config_loader import (
load_json_config,
resolve_config_path,
)
+from openviking_cli.utils.uri import VikingURI
# Error code to exception class mapping
ERROR_CODE_TO_EXCEPTION = {
@@ -281,6 +282,7 @@ class AsyncHTTPClient(BaseClient):
node_limit: int = 1000,
) -> List[Any]:
"""List directory contents."""
+ uri = VikingURI.normalize(uri)
response = await self._http.get(
"/api/v1/fs/ls",
params={
@@ -304,6 +306,7 @@ class AsyncHTTPClient(BaseClient):
node_limit: int = 1000,
) -> List[Dict[str, Any]]:
"""Get directory tree."""
+ uri = VikingURI.normalize(uri)
response = await self._http.get(
"/api/v1/fs/tree",
params={
@@ -318,6 +321,7 @@ class AsyncHTTPClient(BaseClient):
async def stat(self, uri: str) -> Dict[str, Any]:
"""Get resource status."""
+ uri = VikingURI.normalize(uri)
response = await self._http.get(
"/api/v1/fs/stat",
params={"uri": uri},
@@ -326,6 +330,7 @@ class AsyncHTTPClient(BaseClient):
async def mkdir(self, uri: str) -> None:
"""Create directory."""
+ uri = VikingURI.normalize(uri)
response = await self._http.post(
"/api/v1/fs/mkdir",
json={"uri": uri},
@@ -334,6 +339,7 @@ class AsyncHTTPClient(BaseClient):
async def rm(self, uri: str, recursive: bool = False) -> None:
"""Remove resource."""
+ uri = VikingURI.normalize(uri)
response = await self._http.request(
"DELETE",
"/api/v1/fs",
@@ -343,6 +349,8 @@ class AsyncHTTPClient(BaseClient):
async def mv(self, from_uri: str, to_uri: str) -> None:
"""Move resource."""
+ from_uri = VikingURI.normalize(from_uri)
+ to_uri = VikingURI.normalize(to_uri)
response = await self._http.post(
"/api/v1/fs/mv",
json={"from_uri": from_uri, "to_uri": to_uri},
@@ -353,6 +361,7 @@ class AsyncHTTPClient(BaseClient):
async def read(self, uri: str) -> str:
"""Read file content."""
+ uri = VikingURI.normalize(uri)
response = await self._http.get(
"/api/v1/content/read",
params={"uri": uri},
@@ -361,6 +370,7 @@ class AsyncHTTPClient(BaseClient):
async def abstract(self, uri: str) -> str:
"""Read L0 abstract."""
+ uri = VikingURI.normalize(uri)
response = await self._http.get(
"/api/v1/content/abstract",
params={"uri": uri},
@@ -369,6 +379,7 @@ class AsyncHTTPClient(BaseClient):
async def overview(self, uri: str) -> str:
"""Read L1 overview."""
+ uri = VikingURI.normalize(uri)
response = await self._http.get(
"/api/v1/content/overview",
params={"uri": uri},
@@ -386,6 +397,8 @@ class AsyncHTTPClient(BaseClient):
filter: Optional[Dict[str, Any]] = None,
) -> FindResult:
"""Semantic search without session context."""
+ if target_uri:
+ target_uri = VikingURI.normalize(target_uri)
response = await self._http.post(
"/api/v1/search/find",
json={
@@ -409,6 +422,8 @@ class AsyncHTTPClient(BaseClient):
filter: Optional[Dict[str, Any]] = None,
) -> FindResult:
"""Semantic search with optional session context."""
+ if target_uri:
+ target_uri = VikingURI.normalize(target_uri)
sid = session_id or (session.session_id if session else None)
response = await self._http.post(
"/api/v1/search/search",
@@ -425,6 +440,7 @@ class AsyncHTTPClient(BaseClient):
async def grep(self, uri: str, pattern: str, case_insensitive: bool = False) -> Dict[str, Any]:
"""Content search with pattern."""
+ uri = VikingURI.normalize(uri)
response = await self._http.post(
"/api/v1/search/grep",
json={
@@ -437,6 +453,7 @@ class AsyncHTTPClient(BaseClient):
async def glob(self, pattern: str, uri: str = "viking://") -> Dict[str, Any]:
"""File pattern matching."""
+ uri = VikingURI.normalize(uri)
response = await self._http.post(
"/api/v1/search/glob",
json={"pattern": pattern, "uri": uri},
@@ -447,6 +464,7 @@ class AsyncHTTPClient(BaseClient):
async def relations(self, uri: str) -> List[Any]:
"""Get relations for a resource."""
+ uri = VikingURI.normalize(uri)
response = await self._http.get(
"/api/v1/relations",
params={"uri": uri},
@@ -455,6 +473,11 @@ class AsyncHTTPClient(BaseClient):
async def link(self, from_uri: str, to_uris: Union[str, List[str]], reason: str = "") -> None:
"""Create link between resources."""
+ from_uri = VikingURI.normalize(from_uri)
+ if isinstance(to_uris, str):
+ to_uris = VikingURI.normalize(to_uris)
+ else:
+ to_uris = [VikingURI.normalize(u) for u in to_uris]
response = await self._http.post(
"/api/v1/relations/link",
json={"from_uri": from_uri, "to_uris": to_uris, "reason": reason},
@@ -463,6 +486,8 @@ class AsyncHTTPClient(BaseClient):
async def unlink(self, from_uri: str, to_uri: str) -> None:
"""Remove link between resources."""
+ from_uri = VikingURI.normalize(from_uri)
+ to_uri = VikingURI.normalize(to_uri)
response = await self._http.request(
"DELETE",
"/api/v1/relations/link",
@@ -512,6 +537,7 @@ class AsyncHTTPClient(BaseClient):
async def export_ovpack(self, uri: str, to: str) -> str:
"""Export context as .ovpack file."""
+ uri = VikingURI.normalize(uri)
response = await self._http.post(
"/api/v1/pack/export",
json={"uri": uri, "to": to},
@@ -527,6 +553,7 @@ class AsyncHTTPClient(BaseClient):
vectorize: bool = True,
) -> str:
"""Import .ovpack file."""
+ parent = VikingURI.normalize(parent)
response = await self._http.post(
"/api/v1/pack/import",
json={
diff --git a/openviking_cli/utils/config/parser_config.py b/openviking_cli/utils/config/parser_config.py
index 14931b6ea..079daf533 100644
--- a/openviking_cli/utils/config/parser_config.py
+++ b/openviking_cli/utils/config/parser_config.py
@@ -229,7 +229,7 @@ class ImageConfig(ParserConfig):
Configuration for image parsing.
Attributes:
- enable_ocr: Whether to perform OCR text extraction
+ enable_ocr: Whether to perform OCR text extraction, not implemented
enable_vlm: Whether to use VLM for visual understanding
ocr_lang: Language for OCR (e.g., "chi_sim", "eng")
vlm_model: VLM model to use (e.g., "gpt-4-vision")
diff --git a/openviking_cli/utils/config/vlm_config.py b/openviking_cli/utils/config/vlm_config.py
index ad1bea8f2..411c7d76c 100644
--- a/openviking_cli/utils/config/vlm_config.py
+++ b/openviking_cli/utils/config/vlm_config.py
@@ -15,17 +15,16 @@ class VLMConfig(BaseModel):
max_retries: int = Field(default=2, description="Maximum retry attempts")
provider: Optional[str] = Field(default=None, description="Provider type")
- backend: Optional[str] = Field(default=None, description="Backend provider (Deprecated, use 'provider' instead)")
+ backend: Optional[str] = Field(
+ default=None, description="Backend provider (Deprecated, use 'provider' instead)"
+ )
providers: Dict[str, Dict[str, Any]] = Field(
default_factory=dict,
- description="Multi-provider configuration, e.g. {'deepseek': {'api_key': 'xxx', 'api_base': 'xxx'}}"
+ description="Multi-provider configuration, e.g. {'deepseek': {'api_key': 'xxx', 'api_base': 'xxx'}}",
)
- default_provider: Optional[str] = Field(
- default=None,
- description="Default provider name"
- )
+ default_provider: Optional[str] = Field(default=None, description="Default provider name")
thinking: bool = Field(default=False, description="Enable thinking mode for VolcEngine models")
@@ -141,6 +140,7 @@ class VLMConfig(BaseModel):
if self._vlm_instance is None:
config_dict = self._build_vlm_config_dict()
from openviking.models.vlm import VLMFactory
+
self._vlm_instance = VLMFactory.create(config_dict)
return self._vlm_instance
@@ -166,13 +166,15 @@ class VLMConfig(BaseModel):
return result
- def get_completion(self, prompt: str) -> str:
+ def get_completion(self, prompt: str, thinking: bool = False) -> str:
"""Get LLM completion."""
- return self.get_vlm_instance().get_completion(prompt)
+ return self.get_vlm_instance().get_completion(prompt, thinking)
- async def get_completion_async(self, prompt: str, max_retries: int = 0) -> str:
+ async def get_completion_async(
+ self, prompt: str, thinking: bool = False, max_retries: int = 0
+ ) -> str:
"""Get LLM completion asynchronously, max_retries=0 means no retry."""
- return await self.get_vlm_instance().get_completion_async(prompt, max_retries)
+ return await self.get_vlm_instance().get_completion_async(prompt, thinking, max_retries)
def is_available(self) -> bool:
"""Check if LLM is configured."""
@@ -182,14 +184,16 @@ class VLMConfig(BaseModel):
self,
prompt: str,
images: list,
+ thinking: bool = False,
) -> str:
"""Get LLM completion with images."""
- return self.get_vlm_instance().get_vision_completion(prompt, images)
+ return self.get_vlm_instance().get_vision_completion(prompt, images, thinking)
async def get_vision_completion_async(
self,
prompt: str,
images: list,
+ thinking: bool = False,
) -> str:
"""Get LLM completion with images asynchronously."""
- return await self.get_vlm_instance().get_vision_completion_async(prompt, images)
+ return await self.get_vlm_instance().get_vision_completion_async(prompt, images, thinking)
diff --git a/openviking_cli/utils/uri.py b/openviking_cli/utils/uri.py
index 9cf6d8565..6f50a3a99 100644
--- a/openviking_cli/utils/uri.py
+++ b/openviking_cli/utils/uri.py
@@ -201,7 +201,7 @@ class VikingURI:
Build a semantic URI based on parent URI.
"""
# Sanitize semantic name for URI
- safe_name = VikingURI._sanitize_segment(semantic_name)
+ safe_name = VikingURI.sanitize_segment(semantic_name)
if not is_leaf:
return f"{parent_uri}/{safe_name}"
@@ -211,11 +211,12 @@ class VikingURI:
return f"{parent_uri}/{safe_name}/{node_id}"
@staticmethod
- def _sanitize_segment(text: str) -> str:
+ def sanitize_segment(text: str) -> str:
"""
Sanitize text for use in URI segment.
- Preserves Chinese characters but replaces special characters.
+ Preserves CJK characters (Chinese, Japanese, Korean) and other common scripts
+ while replacing special characters.
Args:
text: Original text
@@ -223,8 +224,18 @@ class VikingURI:
Returns:
URI-safe string
"""
- # Preserve Chinese characters, letters, numbers, underscores, hyphens
- safe = re.sub(r"[^\w\u4e00-\u9fff\-]", "_", text)
+ # Preserve:
+ # - Letters, numbers, underscores, hyphens (\w includes [a-zA-Z0-9_])
+ # - CJK Unified Ideographs (Chinese, Japanese Kanji, Korean Hanja)
+ # - Hiragana and Katakana (Japanese)
+ # - Hangul Syllables (Korean)
+ # - CJK Unified Ideographs Extension A
+ # - CJK Unified Ideographs Extension B
+ safe = re.sub(
+ r"[^\w\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af\u3400-\u4dbf\U00020000-\U0002a6df\-]",
+ "_",
+ text,
+ )
# Merge consecutive underscores
safe = re.sub(r"_+", "_", safe)
# Strip leading/trailing underscores and limit length
@@ -245,6 +256,33 @@ class VikingURI:
def __hash__(self) -> int:
return hash(self.uri)
+ @staticmethod
+ def normalize(uri: str) -> str:
+ """
+ Normalize URI by ensuring it has the viking:// scheme.
+
+ If the input already starts with viking://, returns it as-is.
+ If it starts with /, prepends viking:// (resulting in viking:///... which is invalid,
+ so we strip leading / first).
+ Otherwise, prepends viking://.
+
+ Examples:
+ "/resources/images" -> "viking://resources/images"
+ "resources/images" -> "viking://resources/images"
+ "viking://resources/images" -> "viking://resources/images"
+
+ Args:
+ uri: Input URI string
+
+ Returns:
+ Normalized URI with viking:// scheme
+ """
+ if uri.startswith(f"{VikingURI.SCHEME}://"):
+ return uri
+ # Strip leading slashes
+ uri = uri.lstrip("/")
+ return f"{VikingURI.SCHEME}://{uri}"
+
@classmethod
def create_temp_uri(cls) -> str:
"""Create temp directory URI like viking://temp/MMDDHHMM_XXXXXX"""
diff --git a/tests/parse/test_add_directory.py b/tests/parse/test_add_directory.py
index 0402adf31..81fd650bd 100644
--- a/tests/parse/test_add_directory.py
+++ b/tests/parse/test_add_directory.py
@@ -180,6 +180,18 @@ def tmp_mixed(tmp_path: Path) -> Path:
return tmp_path
+def tmp_media_files(tmp_path: Path) -> Path:
+ """Directory with various media files and regular files."""
+ (tmp_path / "docs.md").write_text("# Documentation", encoding="utf-8")
+ (tmp_path / "image.png").write_bytes(b"\x89PNG\r\n\x1a\n")
+ (tmp_path / "photo.jpg").write_bytes(b"\xff\xd8\xff")
+ (tmp_path / "audio.mp3").write_bytes(b"ID3")
+ (tmp_path / "video.mp4").write_bytes(b"\x00\x00\x00\x18ftyp")
+ (tmp_path / "script.js").write_text("console.log('test')", encoding="utf-8")
+ return tmp_path
+
+
# ---------------------------------------------------------------------------
# Tests: basic properties
# ---------------------------------------------------------------------------
@@ -461,3 +473,92 @@ class TestParseResultMetadata:
assert result.meta["dir_name"] == tmp_code.name
assert result.meta["total_processable"] == 3
assert result.meta["file_count"] == 3
+
+
+# ---------------------------------------------------------------------------
+# Tests: directly_upload_media parameter
+# ---------------------------------------------------------------------------
+
+
+class TestDirectlyUploadMedia:
+ """Test the directly_upload_media parameter behavior."""
+
+ @pytest.mark.asyncio
+ async def test_default_directly_upload_media_true(self, tmp_media_files: Path, fake_fs) -> None:
+ """Test that with directly_upload_media=True (default), media files are uploaded directly."""
+ with patch.object(BaseParser, "_get_viking_fs", return_value=fake_fs):
+ parser = DirectoryParser()
+ await parser.parse(str(tmp_media_files))
+
+ uploaded_names = {uri.split("/")[-1] for uri in fake_fs.files}
+
+ assert "docs.md" in uploaded_names
+ assert "image.png" in uploaded_names
+ assert "photo.jpg" in uploaded_names
+ assert "audio.mp3" in uploaded_names
+ assert "video.mp4" in uploaded_names
+ assert "script.js" in uploaded_names
+
+ @pytest.mark.asyncio
+ async def test_directly_upload_media_false(self, tmp_media_files: Path, fake_fs) -> None:
+ """Test that with directly_upload_media=False, media files go through their parsers."""
+ mock_image_result = create_parse_result(
+ root=ResourceNode(type=NodeType.ROOT),
+ source_path=str(tmp_media_files / "image.png"),
+ source_format="image",
+ parser_name="ImageParser",
+ parse_time=0.1,
+ )
+ mock_image_result.temp_dir_path = fake_fs.create_temp_uri()
+
+ mock_audio_result = create_parse_result(
+ root=ResourceNode(type=NodeType.ROOT),
+ source_path=str(tmp_media_files / "audio.mp3"),
+ source_format="audio",
+ parser_name="AudioParser",
+ parse_time=0.1,
+ )
+ mock_audio_result.temp_dir_path = fake_fs.create_temp_uri()
+
+ mock_video_result = create_parse_result(
+ root=ResourceNode(type=NodeType.ROOT),
+ source_path=str(tmp_media_files / "video.mp4"),
+ source_format="video",
+ parser_name="VideoParser",
+ parse_time=0.1,
+ )
+ mock_video_result.temp_dir_path = fake_fs.create_temp_uri()
+
+ with patch.object(BaseParser, "_get_viking_fs", return_value=fake_fs):
+ parser = DirectoryParser()
+
+ with patch.object(parser, "_assign_parser") as mock_assign:
+ from openviking.parse.parsers.media.audio import AudioParser
+ from openviking.parse.parsers.media.image import ImageParser
+ from openviking.parse.parsers.media.video import VideoParser
+
+ mock_image = AsyncMock(spec=ImageParser)
+ mock_image.parse = AsyncMock(return_value=mock_image_result)
+
+ mock_audio = AsyncMock(spec=AudioParser)
+ mock_audio.parse = AsyncMock(return_value=mock_audio_result)
+
+ mock_video = AsyncMock(spec=VideoParser)
+ mock_video.parse = AsyncMock(return_value=mock_video_result)
+
+ def assign_side_effect(cf, registry):
+ if cf.path.suffix in {".png", ".jpg"}:
+ return mock_image
+ elif cf.path.suffix in {".mp3"}:
+ return mock_audio
+ elif cf.path.suffix in {".mp4"}:
+ return mock_video
+ return registry.get_parser_for_file(cf.path)
+
+ mock_assign.side_effect = assign_side_effect
+
+ await parser.parse(str(tmp_media_files), directly_upload_media=False)
+
+ assert mock_image.parse.call_count == 2
+ mock_audio.parse.assert_called_once()
+ mock_video.parse.assert_called_once()
diff --git a/tests/parse/test_directory_parser_routing.py b/tests/parse/test_directory_parser_routing.py
index 88ae61bb2..349fd2f9f 100644
--- a/tests/parse/test_directory_parser_routing.py
+++ b/tests/parse/test_directory_parser_routing.py
@@ -174,8 +174,7 @@ class TestParserSelection:
for ext in self.TEXT_FALLBACK_EXTENSIONS:
assert ext in processable_exts, f"{ext} should be processable (text-fallback)"
- # .bmp and .rar are unsupported
- assert ".bmp" in unsupported_exts
+ # .rar are unsupported
assert ".rar" in unsupported_exts
def test_each_processable_file_has_a_parser_or_is_text(