Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(retrieval): support image search (#3093)

Qin Haojie3003ed61d7b8
diff --git a/Cargo.lock b/Cargo.lock
index 352657a3d..ed90581e7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3401,6 +3401,7 @@ name = "ov_cli"
version = "0.0.0"
dependencies = [
"anyhow",
+ "base64",
"bytes",
"chrono",
"clap",
diff --git a/crates/ov_cli/Cargo.toml b/crates/ov_cli/Cargo.toml
index fb0e599ef..bb33357cc 100644
--- a/crates/ov_cli/Cargo.toml
+++ b/crates/ov_cli/Cargo.toml
@@ -22,6 +22,7 @@ futures = "0.3"
colored = "2.1"
dirs = "5.0"
anyhow = "1.0"
+base64 = "0.22"
chrono = "0.4"
mime_guess = "2.0"
thiserror = "1.0"
diff --git a/crates/ov_cli/src/client.rs b/crates/ov_cli/src/client.rs
index c72be25c5..a9d96efb6 100644
--- a/crates/ov_cli/src/client.rs
+++ b/crates/ov_cli/src/client.rs
@@ -3,6 +3,8 @@ use serde_json::{Map, Value};
use std::env;
use std::path::Path;
+use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
+
pub use crate::base_client::{BaseClient, FileUploader, TimeoutConfig};
use crate::error::{Error, Result};
@@ -33,6 +35,32 @@ fn compact_request_body(body: &mut Value) {
});
}
+fn normalize_image_input(image: Option<String>) -> Result<Option<String>> {
+ let Some(value) = image else {
+ return Ok(None);
+ };
+ if value.starts_with("data:image/")
+ || value.starts_with("http://")
+ || value.starts_with("https://")
+ || value.starts_with("viking://")
+ {
+ return Ok(Some(value));
+ }
+
+ let path = Path::new(&value);
+ if path.is_file() {
+ let bytes = std::fs::read(path)?;
+ let mime = mime_guess::from_path(path).first_or_octet_stream();
+ return Ok(Some(format!(
+ "data:{};base64,{}",
+ mime,
+ BASE64_STANDARD.encode(bytes)
+ )));
+ }
+
+ Ok(Some(value))
+}
+
#[derive(serde::Serialize)]
pub struct SnapshotCommitReq {
pub message: String,
@@ -475,6 +503,7 @@ impl HttpClient {
&self,
query: String,
uri: String,
+ image: Option<String>,
node_limit: i32,
threshold: Option<f64>,
since: Option<String>,
@@ -484,8 +513,10 @@ impl HttpClient {
context_type: Option<Vec<String>>,
tags: Option<Vec<String>>,
) -> Result<serde_json::Value> {
+ let image_url = normalize_image_input(image)?;
let mut body = serde_json::json!({
"query": query,
+ "image_url": image_url,
"target_uri": uri,
"limit": node_limit,
"score_threshold": threshold,
@@ -504,6 +535,7 @@ impl HttpClient {
&self,
query: String,
uri: String,
+ image: Option<String>,
session_id: Option<String>,
node_limit: i32,
threshold: Option<f64>,
@@ -514,8 +546,10 @@ impl HttpClient {
context_type: Option<Vec<String>>,
tags: Option<Vec<String>>,
) -> Result<serde_json::Value> {
+ let image_url = normalize_image_input(image)?;
let mut body = serde_json::json!({
"query": query,
+ "image_url": image_url,
"target_uri": uri,
"session_id": session_id,
"limit": node_limit,
diff --git a/crates/ov_cli/src/commands/search.rs b/crates/ov_cli/src/commands/search.rs
index 4cfb60bff..e57fd07c0 100644
--- a/crates/ov_cli/src/commands/search.rs
+++ b/crates/ov_cli/src/commands/search.rs
@@ -116,6 +116,7 @@ pub async fn find(
client: &HttpClient,
query: &str,
uri: &str,
+ image: Option<String>,
node_limit: i32,
threshold: Option<f64>,
since: Option<&str>,
@@ -131,6 +132,7 @@ pub async fn find(
.find(
query.to_string(),
uri.to_string(),
+ image,
node_limit,
threshold,
since.map(|s| s.to_string()),
@@ -154,6 +156,7 @@ pub async fn search(
client: &HttpClient,
query: &str,
uri: &str,
+ image: Option<String>,
session_id: Option<String>,
node_limit: i32,
threshold: Option<f64>,
@@ -170,6 +173,7 @@ pub async fn search(
.search(
query.to_string(),
uri.to_string(),
+ image,
session_id,
node_limit,
threshold,
diff --git a/crates/ov_cli/src/handlers.rs b/crates/ov_cli/src/handlers.rs
index 4db9559a6..8633e789b 100644
--- a/crates/ov_cli/src/handlers.rs
+++ b/crates/ov_cli/src/handlers.rs
@@ -1256,8 +1256,9 @@ pub async fn handle_get(uri: String, local_path: String, ctx: CliContext) -> Res
}
pub async fn handle_find(
- query: String,
+ query: Option<String>,
uri: String,
+ image: Option<String>,
node_limit: i32,
threshold: Option<f64>,
after: Option<String>,
@@ -1267,7 +1268,16 @@ pub async fn handle_find(
tags: Option<Vec<String>>,
ctx: CliContext,
) -> Result<()> {
+ let query = query.unwrap_or_default();
+ if query.trim().is_empty() && image.is_none() {
+ return Err(Error::Client(
+ "Search query or --image must not be empty.".to_string(),
+ ));
+ }
let mut params = vec![format!("--uri={}", uri), format!("-n {}", node_limit)];
+ if let Some(ref img) = image {
+ params.push(format!("--image {}", img));
+ }
if let Some(t) = threshold {
params.push(format!("--threshold {}", t));
}
@@ -1294,6 +1304,7 @@ pub async fn handle_find(
&client,
&query,
&uri,
+ image,
node_limit,
threshold,
after.as_deref(),
@@ -1309,8 +1320,9 @@ pub async fn handle_find(
}
pub async fn handle_search(
- query: String,
+ query: Option<String>,
uri: String,
+ image: Option<String>,
session_id: Option<String>,
node_limit: i32,
threshold: Option<f64>,
@@ -1321,7 +1333,16 @@ pub async fn handle_search(
tags: Option<Vec<String>>,
ctx: CliContext,
) -> Result<()> {
+ let query = query.unwrap_or_default();
+ if query.trim().is_empty() && image.is_none() {
+ return Err(Error::Client(
+ "Search query or --image must not be empty.".to_string(),
+ ));
+ }
let mut params = vec![format!("--uri={}", uri), format!("-n {}", node_limit)];
+ if let Some(ref img) = image {
+ params.push(format!("--image {}", img));
+ }
if let Some(s) = &session_id {
params.push(format!("--session-id {}", s));
}
@@ -1351,6 +1372,7 @@ pub async fn handle_search(
&client,
&query,
&uri,
+ image,
session_id,
node_limit,
threshold,
diff --git a/crates/ov_cli/src/help_ui.rs b/crates/ov_cli/src/help_ui.rs
index 0690672ae..61295e720 100644
--- a/crates/ov_cli/src/help_ui.rs
+++ b/crates/ov_cli/src/help_ui.rs
@@ -455,6 +455,10 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[
label: "ov find \"auth flow\" -u viking://projects/acme -L 1,2",
description: "Search a subtree and include overview/file results.",
},
+ HelpItem {
+ label: "ov find --image ./query.png -u viking://resources/images",
+ description: "Search by image with a local file or image URI.",
+ },
],
next_steps: &[
HelpItem {
@@ -2818,7 +2822,7 @@ mod tests {
);
assert!(rendered.contains("OpenViking v"));
- assert!(rendered.contains("ov find [OPTIONS] <query>"));
+ assert!(rendered.contains("ov find [OPTIONS] [query]"));
assert!(rendered.contains("Examples"));
assert!(rendered.contains("Common options"));
assert!(rendered.contains("Next"));
@@ -2881,7 +2885,7 @@ mod tests {
);
assert!(rendered.contains("OpenViking v"));
- assert!(rendered.contains("ov find [OPTIONS] <query>"));
+ assert!(rendered.contains("ov find [OPTIONS] [query]"));
assert!(rendered.contains("Usage:"));
}
diff --git a/crates/ov_cli/src/main.rs b/crates/ov_cli/src/main.rs
index 36b9f9a16..6f5bb3433 100644
--- a/crates/ov_cli/src/main.rs
+++ b/crates/ov_cli/src/main.rs
@@ -580,7 +580,14 @@ enum Commands {
Find {
/// Search query
#[arg(value_name = "query")]
- query: String,
+ query: Option<String>,
+ /// Image query: local path, data URI, HTTP URL, or viking:// URI
+ #[arg(
+ long = "image",
+ value_name = "path|uri",
+ help_heading = "Common options"
+ )]
+ image: Option<String>,
/// Target URI
#[arg(
short,
@@ -638,7 +645,14 @@ enum Commands {
Search {
/// Search query
#[arg(value_name = "query")]
- query: String,
+ query: Option<String>,
+ /// Image query: local path, data URI, HTTP URL, or viking:// URI
+ #[arg(
+ long = "image",
+ value_name = "path|uri",
+ help_heading = "Common options"
+ )]
+ image: Option<String>,
/// Target URI
#[arg(
short,
@@ -3096,6 +3110,7 @@ async fn main() {
Commands::Get { uri, local_path } => handlers::handle_get(uri, local_path, ctx).await,
Commands::Find {
query,
+ image,
uri,
node_limit,
threshold,
@@ -3108,6 +3123,7 @@ async fn main() {
handlers::handle_find(
query,
uri,
+ image,
node_limit,
threshold,
after,
@@ -3121,6 +3137,7 @@ async fn main() {
}
Commands::Search {
query,
+ image,
uri,
session_id,
node_limit,
@@ -3134,6 +3151,7 @@ async fn main() {
handlers::handle_search(
query,
uri,
+ image,
session_id,
node_limit,
threshold,
@@ -3237,6 +3255,20 @@ mod tests {
}
}
+ #[test]
+ fn cli_parses_find_image_without_query() {
+ let cli = Cli::try_parse_from(["ov", "find", "--image", "cat.png"])
+ .expect("find image should parse");
+
+ match cli.command {
+ Commands::Find { query, image, .. } => {
+ assert_eq!(query, None);
+ assert_eq!(image.as_deref(), Some("cat.png"));
+ }
+ _ => panic!("expected find command"),
+ }
+ }
+
#[test]
fn cli_parses_admin_migrate_cleanup_flag() {
let migrate = Cli::try_parse_from(["ov", "--sudo", "admin", "migrate"])
@@ -3275,6 +3307,20 @@ mod tests {
}
}
+ #[test]
+ fn cli_parses_search_image_with_query() {
+ let cli = Cli::try_parse_from(["ov", "search", "poster", "--image", "viking://x.png"])
+ .expect("search image should parse");
+
+ match cli.command {
+ Commands::Search { query, image, .. } => {
+ assert_eq!(query.as_deref(), Some("poster"));
+ assert_eq!(image.as_deref(), Some("viking://x.png"));
+ }
+ _ => panic!("expected search command"),
+ }
+ }
+
#[test]
fn cli_find_and_search_reject_removed_peer_id_flag() {
assert!(Cli::try_parse_from(["ov", "find", "invoice", "--peer-id", "peer-a"]).is_err());
diff --git a/docs/en/api/06-retrieval.md b/docs/en/api/06-retrieval.md
index 8802a7ec0..292aba745 100644
--- a/docs/en/api/06-retrieval.md
+++ b/docs/en/api/06-retrieval.md
@@ -54,7 +54,8 @@ The `find()` method performs pure vector similarity search for simple query scen
| Parameter | Type | Required | Default | Description |
|-----------|------|----------|---------|-------------|
-| query | str | Yes | - | Search query string |
+| query | str | No | "" | Search query string. Required unless `image_url` is provided |
+| image_url | str | No | None | Image query as a `data:image/...;base64,...`, `http(s)://`, or `viking://` URI. Requires a multimodal embedding model |
| target_uri | str \| List[str] | No | "" | Limit search to specific URI prefix |
| context_type | str \| List[str] | No | None | Limit results to one or more `ContextType` values: `memory`, `resource`, or `skill` |
| node_limit | int | No | None | Maximum number of results |
@@ -72,6 +73,11 @@ The `find()` method performs pure vector similarity search for simple query scen
- To filter the current user's peer collection to one peer for filesystem and retrieval operations, send `X-OpenViking-Actor-Peer: <peer_id>` or construct the SDK/CLI client with `actor_peer_id`. See [Multi-Tenant: Peer Collection Filter](../concepts/11-multi-tenant.md#peer-restricted-view).
- Current-user shorthand target URIs such as `viking://user/memories`, `viking://user/resources`, and `viking://user/skills` are canonicalized from the authenticated request identity.
+**Image search notes**:
+- Image queries use the image vector as the query and search L2 resource leaf nodes in the target scope by default. Results are not limited to image files; multimodal embedding decides similarity between the query image and text/image resources.
+- Text-only embedding models still index image summaries, but image query input is rejected.
+- Existing image resources keep their existing vectors; image-vector recall applies to images vectorized after this capability is enabled or after a later reindex.
+
**FindResult Structure**
```python
@@ -140,6 +146,18 @@ curl -X POST http://localhost:1933/api/v1/search/find \
-d '{
"query": "authentication",
"context_type": ["memory", "resource"]
+}'
+```
+
+**Image Search**
+
+```bash
+curl -X POST http://localhost:1933/api/v1/search/find \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-key" \
+ -d '{
+ "image_url": "viking://resources/images/cat.png",
+ "limit": 10
}'
```
@@ -169,6 +187,9 @@ typed_results = client.find(
context_type=[ContextType.MEMORY, ContextType.RESOURCE],
)
+# Search by local image, bytes, data URI, HTTP URL, or viking:// URI
+image_results = client.find(image="/path/to/photo.png")
+
# Iterate through results
for ctx in results.resources:
print(f"URI: {ctx.uri}")
@@ -259,6 +280,18 @@ openviking find "how to authenticate users" --level 0
# Limit to specific level(s) (L1 and L2) using short option
openviking find "how to authenticate users" -L 1,2
+
+# Image queries use only --image; pass a local path, viking://, http(s)://, or data:image URI
+openviking find --image ./query.png --uri "viking://resources/images" --limit 5
+
+# Search by an image already stored in VikingFS
+openviking find --image "viking://resources/images/cat.png" --uri "viking://resources/images" --limit 5
+
+# Search by a public image URL
+openviking find --image "https://example.com/images/cat.png" --uri "viking://resources/images" --limit 5
+
+# Combine text and image
+openviking find "red poster style" --image ./poster.png --uri "viking://resources/images"
```
**Response Example**
@@ -327,7 +360,8 @@ The `search()` method adds session context understanding and intent analysis cap
| Parameter | Type | Required | Default | Description |
|-----------|------|----------|---------|-------------|
-| query | str | Yes | - | Search query string |
+| query | str | No | "" | Search query string. Required unless `image_url` is provided |
+| image_url | str | No | None | Image query as a `data:image/...;base64,...`, `http(s)://`, or `viking://` URI. Requires a multimodal embedding model |
| target_uri | str \| List[str] | No | "" | Limit search to specific URI prefix |
| session | Session | No | None | Session for context-aware search (SDK) |
| session_id | str | No | None | Session ID for context-aware search (HTTP) |
@@ -342,7 +376,7 @@ The `search()` method adds session context understanding and intent analysis cap
| include_provenance | bool | No | False | Include provenance/query-plan details in serialized result |
| telemetry | bool \| object | No | False | Attach telemetry data to response |
-`search()` uses the same target resolution rules as `find()`, including the peer collection filter selected by `X-OpenViking-Actor-Peer` or SDK `actor_peer_id`.
+`search()` uses the same target resolution rules as `find()`, including the peer collection filter selected by `X-OpenViking-Actor-Peer` or SDK `actor_peer_id`. When `image_url` is provided, `search()` uses direct image retrieval and skips session query planning.
#### 3. Usage Examples
@@ -374,6 +408,19 @@ curl -X POST http://localhost:1933/api/v1/search/search \
-H "X-API-Key: your-key" \
-d '{
"query": "how to implement OAuth 2.0 authorization code flow"
+}'
+```
+
+**Image Search**
+
+```bash
+curl -X POST http://localhost:1933/api/v1/search/search \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-key" \
+ -d '{
+ "query": "similar poster",
+ "image_url": "data:image/png;base64,...",
+ "limit": 10
}'
```
@@ -422,6 +469,12 @@ for ctx in results.resources:
print(f"Found: {ctx.uri} (score: {ctx.score:.3f})")
```
+**Image Search**
+
+```python
+results = client.search("similar poster", image="/path/to/poster.png")
+```
+
**Go SDK**
```go
@@ -456,6 +509,9 @@ openviking search "best practices" --level 0
# Limit to specific level(s) (L1 and L2) using short option
openviking search "how to implement OAuth" -L 1,2
+
+# Image queries also use --image; they use direct retrieval and skip session planning
+openviking search "similar poster" --image ./poster.png --uri "viking://resources/images"
```
**Response Example**
diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md
index 67a4080a3..f841e10a1 100644
--- a/docs/en/guides/01-configuration.md
+++ b/docs/en/guides/01-configuration.md
@@ -253,7 +253,7 @@ When the embedding provider experiences consecutive transient failures (e.g. `42
| `doubao-embedding-vision-251215` | 1024 | multimodal | Recommended |
| `doubao-embedding-250615` | 1024 | text | Text only |
-With `input: "multimodal"`, OpenViking can embed text, images (PNG, JPG, etc.), and mixed content.
+With `input: "multimodal"`, OpenViking can embed text, images (PNG, JPG, etc.), and mixed content. Image-to-image search requires this mode; text-only embedding models continue to index image summaries but cannot accept image queries.
**Supported providers:**
- `openai`: OpenAI Embedding API
diff --git a/docs/zh/api/06-retrieval.md b/docs/zh/api/06-retrieval.md
index dafa6b06d..affc8a8c1 100644
--- a/docs/zh/api/06-retrieval.md
+++ b/docs/zh/api/06-retrieval.md
@@ -54,7 +54,8 @@ OpenViking 提供多种检索方法,包括简单的向量相似度搜索、带
| 参数 | 类型 | 必填 | 默认值 | 说明 |
|------|------|------|--------|------|
-| query | str | 是 | - | 搜索查询字符串 |
+| query | str | 否 | "" | 搜索查询字符串;未提供 `image_url` 时必填 |
+| image_url | str | 否 | None | 图片查询,支持 `data:image/...;base64,...`、`http(s)://` 或 `viking://` URI;需要 multimodal embedding 模型 |
| target_uri | str \| List[str] | 否 | "" | 限制搜索范围到指定的 URI 前缀 |
| context_type | str \| List[str] | 否 | None | 限定一个或多个 `ContextType` 取值:`memory`、`resource` 或 `skill` |
| limit | int | 否 | 10 | 最大返回结果数 |
@@ -73,6 +74,11 @@ OpenViking 提供多种检索方法,包括简单的向量相似度搜索、带
- 如需在文件系统和检索操作中把当前用户的 peer 集合过滤到某一个 peer,发送 `X-OpenViking-Actor-Peer: <peer_id>`,或用 SDK/CLI client 的 `actor_peer_id` 初始化。见 [多租户:Peer 集合过滤](../concepts/11-multi-tenant.md#peer-restricted-view)。
- `viking://user/memories`、`viking://user/resources`、`viking://user/skills` 等当前用户短写 target URI 会按认证请求身份 canonicalize。
+**图片搜索说明**:
+- 图片查询会以图片向量作为 query,默认检索目标范围内的 L2 resource 叶子节点;结果不限于图片文件,图片与文本/图片资源的相似度由 multimodal embedding 模型决定。
+- 纯文本 embedding 模型仍会索引图片 summary,但会拒绝图片查询输入。
+- 已有图片资源保持现有向量不变;图片向量召回只作用于开启该能力后向量化的图片,或之后手动 reindex 的图片。
+
**FindResult 结构**
```python
@@ -141,6 +147,18 @@ curl -X POST http://localhost:1933/api/v1/search/find \
-d '{
"query": "authentication",
"context_type": ["memory", "resource"]
+}'
+```
+
+**图片搜索**
+
+```bash
+curl -X POST http://localhost:1933/api/v1/search/find \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-key" \
+ -d '{
+ "image_url": "viking://resources/images/cat.png",
+ "limit": 10
}'
```
@@ -170,6 +188,9 @@ typed_results = client.find(
context_type=[ContextType.MEMORY, ContextType.RESOURCE],
)
+# 按本地图片、bytes、data URI、HTTP URL 或 viking:// URI 搜索
+image_results = client.find(image="/path/to/photo.png")
+
# 遍历结果
for ctx in results.resources:
print(f"URI: {ctx.uri}")
@@ -260,6 +281,18 @@ openviking find "how to authenticate users" --level 0
# 限定层级范围 (L1 和 L2),使用短选项
openviking find "how to authenticate users" -L 1,2
+
+# 图片查询统一使用 --image;可传本地路径、viking://、http(s):// 或 data:image URI
+openviking find --image ./query.png --uri "viking://resources/images" --limit 5
+
+# 使用已入库图片搜索
+openviking find --image "viking://resources/images/cat.png" --uri "viking://resources/images" --limit 5
+
+# 使用公网图片 URL 搜索
+openviking find --image "https://example.com/images/cat.png" --uri "viking://resources/images" --limit 5
+
+# 图文联合检索
+openviking find "红色海报风格" --image ./poster.png --uri "viking://resources/images"
```
**响应示例**
@@ -328,7 +361,8 @@ openviking find "how to authenticate users" -L 1,2
| 参数 | 类型 | 必填 | 默认值 | 说明 |
|------|------|------|--------|------|
-| query | str | 是 | - | 搜索查询字符串 |
+| query | str | 否 | "" | 搜索查询字符串;未提供 `image_url` 时必填 |
+| image_url | str | 否 | None | 图片查询,支持 `data:image/...;base64,...`、`http(s)://` 或 `viking://` URI;需要 multimodal embedding 模型 |
| target_uri | str \| List[str] | 否 | "" | 限制搜索范围到指定的 URI 前缀 |
| session | Session | 否 | None | 用于上下文感知搜索的会话(SDK)|
| session_id | str | 否 | None | 用于上下文感知搜索的会话 ID(HTTP)|
@@ -344,7 +378,7 @@ openviking find "how to authenticate users" -L 1,2
| include_provenance | bool | 否 | False | 在序列化结果中附带 provenance / query-plan 细节 |
| telemetry | bool \| object | 否 | False | 在响应中附带遥测数据 |
-`search()` 使用和 `find()` 相同的目标解析规则,包括由 `X-OpenViking-Actor-Peer` 或 SDK `actor_peer_id` 选择的 peer 集合过滤。
+`search()` 使用和 `find()` 相同的目标解析规则,包括由 `X-OpenViking-Actor-Peer` 或 SDK `actor_peer_id` 选择的 peer 集合过滤。提供 `image_url` 时,`search()` 会直接执行图片检索并跳过会话 query planning。
#### 3. 使用示例
@@ -376,6 +410,19 @@ curl -X POST http://localhost:1933/api/v1/search/search \
-H "X-API-Key: your-key" \
-d '{
"query": "how to implement OAuth 2.0 authorization code flow"
+}'
+```
+
+**图片搜索**
+
+```bash
+curl -X POST http://localhost:1933/api/v1/search/search \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-key" \
+ -d '{
+ "query": "similar poster",
+ "image_url": "data:image/png;base64,...",
+ "limit": 10
}'
```
@@ -424,6 +471,12 @@ for ctx in results.resources:
print(f"Found: {ctx.uri} (score: {ctx.score:.3f})")
```
+**图片搜索**
+
+```python
+results = client.search("similar poster", image="/path/to/poster.png")
+```
+
**Go SDK**
```go
@@ -458,6 +511,9 @@ openviking search "best practices" --level 0
# 限定层级范围(L1 和 L2),使用短选项
openviking search "how to implement OAuth" -L 1,2
+
+# 图片查询同样使用 --image;会直接检索并跳过 session planning
+openviking search "similar poster" --image ./poster.png --uri "viking://resources/images"
```
**响应示例**
diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md
index bfe108078..4f9c8f569 100644
--- a/docs/zh/guides/01-configuration.md
+++ b/docs/zh/guides/01-configuration.md
@@ -255,7 +255,7 @@ openviking-server doctor
| `doubao-embedding-vision-251215` | 1024 | multimodal | 推荐 |
| `doubao-embedding-250615` | 1024 | text | 仅文本 |
-使用 `input: "multimodal"` 时,OpenViking 可以嵌入文本、图片(PNG、JPG 等)和混合内容。
+使用 `input: "multimodal"` 时,OpenViking 可以嵌入文本、图片(PNG、JPG 等)和混合内容。以图搜图需要该模式;纯文本 embedding 模型仍会索引图片 summary,但不能接收图片查询。
**支持的 provider:**
- `openai`: OpenAI Embedding API
diff --git a/openviking/async_client.py b/openviking/async_client.py
index 2bbece85a..c2c3792a8 100644
--- a/openviking/async_client.py
+++ b/openviking/async_client.py
@@ -9,7 +9,7 @@ For HTTP mode, use AsyncHTTPClient or SyncHTTPClient.
from __future__ import annotations
import threading
-from typing import Any, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from openviking.client import LocalClient, Session
from openviking.service.debug_service import SystemStatus
@@ -22,6 +22,10 @@ from openviking_cli.utils import get_logger
logger = get_logger(__name__)
+if TYPE_CHECKING:
+ from openviking.snapshot_namespace import AsyncSnapshotNamespace
+
+
class AsyncOpenViking:
"""
OpenViking main client class (Asynchronous, embedded mode only).
@@ -510,7 +514,7 @@ class AsyncOpenViking:
async def search(
self,
- query: str,
+ query: str = "",
target_uri: Union[str, List[str]] = "",
session: Optional[Union["Session", Any]] = None,
session_id: Optional[str] = None,
@@ -524,6 +528,7 @@ class AsyncOpenViking:
until: Optional[str] = None,
time_field: Optional[str] = None,
level: Optional[List[int]] = None,
+ image: Optional[Any] = None,
):
"""
Complex search with session context.
@@ -555,11 +560,12 @@ class AsyncOpenViking:
until=until,
time_field=time_field,
level=level,
+ image=image,
)
async def find(
self,
- query: str,
+ query: str = "",
target_uri: Union[str, List[str]] = "",
limit: int = 10,
score_threshold: Optional[float] = None,
@@ -571,6 +577,7 @@ class AsyncOpenViking:
until: Optional[str] = None,
time_field: Optional[str] = None,
level: Optional[List[int]] = None,
+ image: Optional[Any] = None,
):
"""Semantic search"""
await self._ensure_initialized()
@@ -587,6 +594,7 @@ class AsyncOpenViking:
until=until,
time_field=time_field,
level=level,
+ image=image,
)
# ============= FS methods =============
diff --git a/openviking/client/local.py b/openviking/client/local.py
index 37dbc070f..93e807328 100644
--- a/openviking/client/local.py
+++ b/openviking/client/local.py
@@ -12,7 +12,6 @@ from openviking.server.identity import RequestContext, Role
from openviking.server.routers.skills import (
_list_skill_files,
_list_skills_from_root,
- _merge_skills,
_require_skill,
_restore_skill_privacy,
_skill_summary_from_hit,
@@ -25,6 +24,7 @@ from openviking.telemetry.execution import (
attach_telemetry_payload,
run_with_telemetry,
)
+from openviking.utils.image_search import normalize_client_image_input
from openviking.utils.search_filters import SearchContextTypeInput, merge_search_filter
from openviking.utils.tags import normalize_search_tags
from openviking_cli.client.base import BaseClient
@@ -290,11 +290,11 @@ class LocalClient(BaseClient):
) -> Dict[str, Any]:
"""Get a skill by name."""
from openviking.server.routers.skills import (
+ SOURCE_METADATA_FILENAME,
_parse_abstract_meta,
_relative_skill_path,
_skill_file_kind,
_skill_summary_from_meta,
- SOURCE_METADATA_FILENAME,
)
from openviking.server.skill_source_metadata import read_skill_source_metadata
@@ -476,7 +476,7 @@ class LocalClient(BaseClient):
service = self._service
ctx = self._ctx
root_uri = await _require_skill(service, ctx, skill_name, target_uri)
- result = await service.fs.rm(root_uri, ctx=ctx, recursive=True)
+ await service.fs.rm(root_uri, ctx=ctx, recursive=True)
return {"name": skill_name, "uri": root_uri, "root_uri": root_uri, "deleted": True}
async def validate_skill(
@@ -674,7 +674,7 @@ class LocalClient(BaseClient):
async def find(
self,
- query: str,
+ query: str = "",
target_uri: Union[str, List[str]] = "",
limit: int = 10,
score_threshold: Optional[float] = None,
@@ -686,11 +686,13 @@ class LocalClient(BaseClient):
until: Optional[str] = None,
time_field: Optional[str] = None,
level: Optional[List[int]] = None,
+ image: Optional[Any] = None,
) -> Any:
"""Semantic search without session context."""
resolved_filter = _resolve_search_filter(
filter, context_type, since, until, time_field, tags
)
+ image_url = normalize_client_image_input(image)
execution = await run_with_telemetry(
operation="search.find",
telemetry=telemetry,
@@ -702,6 +704,7 @@ class LocalClient(BaseClient):
score_threshold=score_threshold,
filter=resolved_filter,
level=level,
+ image_url=image_url,
),
)
return attach_telemetry_payload(
@@ -711,7 +714,7 @@ class LocalClient(BaseClient):
async def search(
self,
- query: str,
+ query: str = "",
target_uri: Union[str, List[str]] = "",
session_id: Optional[str] = None,
limit: int = 10,
@@ -724,11 +727,13 @@ class LocalClient(BaseClient):
until: Optional[str] = None,
time_field: Optional[str] = None,
level: Optional[List[int]] = None,
+ image: Optional[Any] = None,
) -> Any:
"""Semantic search with optional session context."""
resolved_filter = _resolve_search_filter(
filter, context_type, since, until, time_field, tags
)
+ image_url = normalize_client_image_input(image)
async def _search():
session = None
@@ -744,6 +749,7 @@ class LocalClient(BaseClient):
score_threshold=score_threshold,
filter=resolved_filter,
level=level,
+ image_url=image_url,
)
execution = await run_with_telemetry(
diff --git a/openviking/retrieve/hierarchical_retriever.py b/openviking/retrieve/hierarchical_retriever.py
index 36ca542b0..de258bd51 100644
--- a/openviking/retrieve/hierarchical_retriever.py
+++ b/openviking/retrieve/hierarchical_retriever.py
@@ -25,6 +25,7 @@ from openviking.storage import VikingDBManager, VikingDBManagerProxy
from openviking.storage.expr import FilterExpr
from openviking.telemetry import get_current_telemetry
from openviking.utils.time_utils import parse_iso_datetime
+from openviking_cli.exceptions import InvalidArgumentError
from openviking_cli.retrieve.types import (
ContextType,
MatchedContext,
@@ -114,8 +115,13 @@ class HierarchicalRetriever:
t0 = time.monotonic()
telemetry = get_current_telemetry()
effective_threshold = self._resolve_threshold(score_threshold)
+ image_query = bool(getattr(query, "image_query", False))
if mode is None:
mode = RetrieverMode.QUICK if not self._rerank_client else RetrieverMode.THINKING
+ if image_query:
+ mode = RetrieverMode.QUICK
+ if level is None:
+ level = [2]
# 创建 proxy 包装器,绑定当前 ctx
vector_proxy = VikingDBManagerProxy(self.vector_store, ctx)
@@ -137,8 +143,17 @@ class HierarchicalRetriever:
query_vector = None
sparse_query_vector = None
if self.embedder:
+ if image_query and not getattr(self.embedder, "supports_multimodal", False):
+ raise InvalidArgumentError(
+ "Image search requires a multimodal embedding model."
+ )
with telemetry.measure("search.embed_query"):
- result: EmbedResult = await embed_compat(self.embedder, query.query, is_query=True)
+ embedding_input = getattr(query, "embedding_input", None) or query.query
+ result: EmbedResult = await embed_compat(
+ self.embedder,
+ embedding_input,
+ is_query=True,
+ )
query_vector = result.dense_vector
sparse_query_vector = result.sparse_vector
@@ -149,8 +164,11 @@ class HierarchicalRetriever:
root_uris = default_target_directories(ctx, context_type=query.context_type)
context_type = query.context_type.value if query.context_type else None
+ if image_query and context_type is None:
+ context_type = ContextType.RESOURCE.value
if mode == RetrieverMode.QUICK:
+ search_limit = max(limit * 5, 50) if image_query else max(limit, self.GLOBAL_SEARCH_TOPK)
with telemetry.measure("search.vector_retrieval"):
quick_results = await vector_proxy.search_in_tenant(
query_vector=query_vector,
@@ -159,7 +177,7 @@ class HierarchicalRetriever:
target_directories=target_dirs,
extra_filter=scope_dsl,
level=level,
- limit=max(limit, self.GLOBAL_SEARCH_TOPK),
+ limit=search_limit,
)
telemetry.count("vector.searches", 1)
telemetry.count("vector.scored", len(quick_results))
diff --git a/openviking/server/routers/search.py b/openviking/server/routers/search.py
index 7155e955b..9b49dbfbc 100644
--- a/openviking/server/routers/search.py
+++ b/openviking/server/routers/search.py
@@ -96,7 +96,8 @@ class FindRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
- query: str
+ query: str = ""
+ image_url: Optional[str] = None
target_uri: Union[str, List[str]] = ""
context_type: Optional[Union[str, List[str]]] = None
agent_id: Optional[str] = None
@@ -119,7 +120,8 @@ class SearchRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
- query: str
+ query: str = ""
+ image_url: Optional[str] = None
target_uri: Union[str, List[str]] = ""
context_type: Optional[Union[str, List[str]]] = None
agent_id: Optional[str] = None
@@ -199,6 +201,7 @@ async def find(
score_threshold=request.score_threshold,
filter=effective_filter,
level=_resolve_levels(request.level) or None,
+ image_url=request.image_url,
),
)
result = execution.result
@@ -244,6 +247,7 @@ async def search(
score_threshold=request.score_threshold,
filter=effective_filter,
level=_resolve_levels(request.level) or None,
+ image_url=request.image_url,
)
execution = await run_operation(
diff --git a/openviking/service/search_service.py b/openviking/service/search_service.py
index f57debb03..58b426ac4 100644
--- a/openviking/service/search_service.py
+++ b/openviking/service/search_service.py
@@ -8,9 +8,16 @@ Provides semantic search operations: search, find.
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+from openviking.core.path_variables import resolve_path_variables
from openviking.core.uri_validation import validate_optional_viking_uris
from openviking.server.identity import RequestContext
from openviking.storage.viking_fs import VikingFS
+from openviking.utils.image_search import (
+ image_bytes_to_data_uri,
+ is_data_image_uri,
+ is_http_url,
+ is_viking_uri,
+)
from openviking_cli.exceptions import InvalidArgumentError, NotInitializedError
from openviking_cli.utils import get_logger
@@ -20,9 +27,9 @@ if TYPE_CHECKING:
logger = get_logger(__name__)
-def _ensure_non_empty_query(query: str) -> None:
- if not query.strip():
- raise InvalidArgumentError("Search query must not be empty.")
+def _ensure_non_empty_query(query: str, image_url: Optional[str] = None) -> None:
+ if not query.strip() and not image_url:
+ raise InvalidArgumentError("Search query or image_url must not be empty.")
class SearchService:
@@ -41,6 +48,24 @@ class SearchService:
raise NotInitializedError("VikingFS")
return self._viking_fs
+ async def _resolve_image_url(
+ self,
+ image_url: Optional[str],
+ ctx: RequestContext,
+ ) -> Optional[str]:
+ if not image_url:
+ return None
+ image_url = resolve_path_variables(image_url)
+ if is_viking_uri(image_url):
+ viking_fs = self._ensure_initialized()
+ content = await viking_fs.read_file_bytes(image_url, ctx=ctx)
+ return image_bytes_to_data_uri(content, image_url)
+ if is_data_image_uri(image_url) or is_http_url(image_url):
+ return image_url
+ raise InvalidArgumentError(
+ "image_url must be a data:image base64 URI, http(s) URL, or viking:// URI."
+ )
+
async def search(
self,
query: str,
@@ -51,6 +76,7 @@ class SearchService:
score_threshold: Optional[float] = None,
filter: Optional[Dict] = None,
level: Optional[List[int]] = None,
+ image_url: Optional[str] = None,
) -> Any:
"""Complex search with session context.
@@ -66,12 +92,13 @@ class SearchService:
Returns:
FindResult
"""
- _ensure_non_empty_query(query)
+ resolved_image_url = await self._resolve_image_url(image_url, ctx)
+ _ensure_non_empty_query(query, resolved_image_url)
target_uri = validate_optional_viking_uris(target_uri, field_name="target_uri")
viking_fs = self._ensure_initialized()
session_info = None
- if session:
+ if session and not resolved_image_url:
session_info = await session.get_context_for_search(query)
result = await viking_fs.search(
@@ -83,6 +110,7 @@ class SearchService:
score_threshold=score_threshold,
filter=filter,
level=level,
+ image_url=resolved_image_url,
)
return result
@@ -95,6 +123,7 @@ class SearchService:
score_threshold: Optional[float] = None,
filter: Optional[Dict] = None,
level: Optional[List[int]] = None,
+ image_url: Optional[str] = None,
) -> Any:
"""Semantic search without session context.
@@ -109,7 +138,8 @@ class SearchService:
Returns:
FindResult
"""
- _ensure_non_empty_query(query)
+ resolved_image_url = await self._resolve_image_url(image_url, ctx)
+ _ensure_non_empty_query(query, resolved_image_url)
target_uri = validate_optional_viking_uris(target_uri, field_name="target_uri")
viking_fs = self._ensure_initialized()
result = await viking_fs.find(
@@ -120,5 +150,6 @@ class SearchService:
score_threshold=score_threshold,
filter=filter,
level=level,
+ image_url=resolved_image_url,
)
return result
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index cdaf2502c..c9c58dec6 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -51,6 +51,7 @@ from openviking.storage.internal_names import (
STORAGE_INTERNAL_ENTRY_NAMES,
)
from openviking.telemetry import get_current_telemetry
+from openviking.utils.image_search import build_multimodal_embedding_input
from openviking.utils.time_utils import format_iso8601, get_current_timestamp, parse_iso_datetime
from openviking_cli.exceptions import (
FailedPreconditionError,
@@ -81,9 +82,9 @@ LS_ALL_NODES = 2**31 - 1
_T = TypeVar("_T")
-def _ensure_non_empty_search_query(query: str) -> None:
- if not query.strip():
- raise InvalidArgumentError("Search query must not be empty.")
+def _ensure_non_empty_search_query(query: str, image_url: Optional[str] = None) -> None:
+ if not query.strip() and not image_url:
+ raise InvalidArgumentError("Search query or image_url must not be empty.")
def _is_directory_not_empty_error(message: str) -> bool:
@@ -1976,6 +1977,7 @@ class VikingFS:
filter: Optional[Dict] = None,
ctx: Optional[RequestContext] = None,
level: Optional[List[int]] = None,
+ image_url: Optional[str] = None,
):
"""Semantic search.
@@ -1989,7 +1991,7 @@ class VikingFS:
Returns:
FindResult
"""
- _ensure_non_empty_search_query(query)
+ _ensure_non_empty_search_query(query, image_url)
telemetry = get_current_telemetry()
from openviking.retrieve.hierarchical_retriever import HierarchicalRetriever
from openviking_cli.retrieve import (
@@ -2024,6 +2026,10 @@ class VikingFS:
context_type=None,
intent="",
target_directories=retrieval_targets.target_directories,
+ embedding_input=(
+ build_multimodal_embedding_input(query, image_url) if image_url else None
+ ),
+ image_query=bool(image_url),
)
logger.debug(
@@ -2068,6 +2074,7 @@ class VikingFS:
filter: Optional[Dict] = None,
ctx: Optional[RequestContext] = None,
level: Optional[List[int]] = None,
+ image_url: Optional[str] = None,
):
"""Complex search with session context.
@@ -2081,7 +2088,7 @@ class VikingFS:
Returns:
FindResult
"""
- _ensure_non_empty_search_query(query)
+ _ensure_non_empty_search_query(query, image_url)
telemetry = get_current_telemetry()
from openviking.retrieve.hierarchical_retriever import HierarchicalRetriever
from openviking.retrieve.intent_analyzer import IntentAnalyzer
@@ -2115,7 +2122,19 @@ class VikingFS:
target_abstract = ""
# With session context: intent analysis
- if session_summary or current_messages:
+ if image_url:
+ typed_queries = [
+ TypedQuery(
+ query=query,
+ context_type=None,
+ intent="",
+ priority=1,
+ target_directories=retrieval_targets.target_directories,
+ embedding_input=build_multimodal_embedding_input(query, image_url),
+ image_query=True,
+ )
+ ]
+ elif session_summary or current_messages:
analyzer = IntentAnalyzer(max_recent_messages=5)
with telemetry.measure("search.intent_analysis"):
query_plan = await analyzer.analyze(
diff --git a/openviking/sync_client.py b/openviking/sync_client.py
index af31adb36..0c23fb3c7 100644
--- a/openviking/sync_client.py
+++ b/openviking/sync_client.py
@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
if TYPE_CHECKING:
from openviking.session import Session
+ from openviking.snapshot_namespace import SyncSnapshotNamespace
from openviking.async_client import AsyncOpenViking
from openviking.telemetry import TelemetryRequest
@@ -373,7 +374,7 @@ class SyncOpenViking:
def search(
self,
- query: str,
+ query: str = "",
target_uri: Union[str, List[str]] = "",
session: Optional["Session"] = None,
session_id: Optional[str] = None,
@@ -387,6 +388,7 @@ class SyncOpenViking:
until: Optional[str] = None,
time_field: Optional[str] = None,
level: Optional[List[int]] = None,
+ image: Optional[Any] = None,
):
"""Execute complex retrieval (intent analysis, hierarchical retrieval)."""
return run_async(
@@ -405,12 +407,13 @@ class SyncOpenViking:
until=until,
time_field=time_field,
level=level,
+ image=image,
)
)
def find(
self,
- query: str,
+ query: str = "",
target_uri: Union[str, List[str]] = "",
limit: int = 10,
score_threshold: Optional[float] = None,
@@ -422,6 +425,7 @@ class SyncOpenViking:
until: Optional[str] = None,
time_field: Optional[str] = None,
level: Optional[List[int]] = None,
+ image: Optional[Any] = None,
):
"""Quick retrieval"""
return run_async(
@@ -438,6 +442,7 @@ class SyncOpenViking:
until,
time_field,
level,
+ image,
)
)
diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.py
index 9c6e41f99..6e531c215 100644
--- a/openviking/utils/embedding_utils.py
+++ b/openviking/utils/embedding_utils.py
@@ -6,7 +6,6 @@ Embedding utilities for OpenViking.
Common logic for creating Context objects and enqueuing them to EmbeddingQueue.
"""
-import base64
import os
from datetime import datetime, timezone
from typing import Any, Dict, Optional
@@ -19,6 +18,7 @@ from openviking.server.identity import RequestContext
from openviking.storage.queuefs import get_queue_manager
from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter
from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs
+from openviking.utils.image_search import image_bytes_to_data_uri
from openviking.utils.time_utils import parse_iso_datetime
from openviking_cli.utils import VikingURI, get_logger
from openviking_cli.utils.config import get_openviking_config
@@ -216,23 +216,6 @@ def get_resource_content_type(file_name: str) -> Optional[ResourceContentType]:
return None
-_IMAGE_MIME_TYPES = {
- ".png": "image/png",
- ".jpg": "image/jpeg",
- ".jpeg": "image/jpeg",
- ".gif": "image/gif",
- ".bmp": "image/bmp",
- ".svg": "image/svg+xml",
- ".webp": "image/webp",
-}
-
-
-def _image_mime_type(file_name: str) -> str:
- """Resolve the MIME type for an image file based on its extension."""
- _, ext = os.path.splitext(file_name.lower())
- return _IMAGE_MIME_TYPES.get(ext, "image/png")
-
-
async def _build_image_data_uri(
file_path: str,
file_name: str,
@@ -245,8 +228,7 @@ async def _build_image_data_uri(
"""
try:
content = await viking_fs.read_file_bytes(file_path, ctx=ctx)
- encoded = base64.b64encode(content).decode("ascii")
- return f"data:{_image_mime_type(file_name)};base64,{encoded}"
+ return image_bytes_to_data_uri(content, file_name)
except Exception as e:
logger.warning(f"Failed to read image for multimodal vectorization {file_path}: {e}")
return None
@@ -490,7 +472,6 @@ async def vectorize_file(
embedding_cfg = get_openviking_config().embedding
configured_text_source = getattr(embedding_cfg, "text_source", "content_only")
effective_text_source = "summary_only" if use_summary else configured_text_source
- image_vectorization = getattr(embedding_cfg, "image_vectorization", "summary_only")
if content_type is None:
# Unsupported file type: fall back to summary if available
@@ -534,17 +515,12 @@ async def vectorize_file(
else:
# Embedders apply their own input guard.
context.set_vectorize(Vectorize(text=content, full_text=content))
- elif content_type == ResourceContentType.IMAGE and image_vectorization in {
- "image_only",
- "image_and_summary",
- }:
- # Multimodal: embed the image itself (optionally with its text summary).
+ elif content_type == ResourceContentType.IMAGE:
+ # Multimodal embedders consume both parts; text-only embedders fall back to summary.
image_uri = await _build_image_data_uri(file_path, file_name, viking_fs, ctx)
if image_uri:
- text = summary if image_vectorization == "image_and_summary" else ""
- context.set_vectorize(Vectorize(text=text, images=[image_uri]))
+ context.set_vectorize(Vectorize(text=summary, images=[image_uri]))
elif summary:
- # Could not load image; fall back to summary text.
context.set_vectorize(Vectorize(text=summary))
else:
logger.debug(
diff --git a/openviking/utils/image_search.py b/openviking/utils/image_search.py
new file mode 100644
index 000000000..e44eb279c
--- /dev/null
+++ b/openviking/utils/image_search.py
@@ -0,0 +1,63 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Small helpers for image search inputs."""
+
+from __future__ import annotations
+
+import base64
+import mimetypes
+import os
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+
+def image_mime_type(file_name: str = "") -> str:
+ mime_type, _ = mimetypes.guess_type(file_name or "")
+ if mime_type and mime_type.startswith("image/"):
+ return mime_type
+ return "image/png"
+
+
+def image_bytes_to_data_uri(data: bytes | bytearray | memoryview, file_name: str = "") -> str:
+ encoded = base64.b64encode(bytes(data)).decode("ascii")
+ return f"data:{image_mime_type(file_name)};base64,{encoded}"
+
+
+def is_data_image_uri(value: str) -> bool:
+ return value.startswith("data:image/") and ";base64," in value
+
+
+def is_http_url(value: str) -> bool:
+ return value.startswith(("http://", "https://"))
+
+
+def is_viking_uri(value: str) -> bool:
+ return value.startswith("viking://")
+
+
+def build_multimodal_embedding_input(
+ text: str = "",
+ image_url: Optional[str] = None,
+) -> List[Dict[str, Any]]:
+ parts: List[Dict[str, Any]] = []
+ if text.strip():
+ parts.append({"type": "text", "text": text})
+ if image_url:
+ parts.append({"type": "image_url", "image_url": {"url": image_url}})
+ return parts
+
+
+def normalize_client_image_input(image: Any) -> Optional[str]:
+ if image is None:
+ return None
+ if isinstance(image, (bytes, bytearray, memoryview)):
+ return image_bytes_to_data_uri(image)
+
+ value = os.fspath(image) if isinstance(image, os.PathLike) else str(image)
+ if is_data_image_uri(value) or is_http_url(value) or is_viking_uri(value):
+ return value
+
+ path = Path(value).expanduser()
+ if path.is_file():
+ return image_bytes_to_data_uri(path.read_bytes(), path.name)
+ return value
diff --git a/openviking_cli/client/base.py b/openviking_cli/client/base.py
index 3be8121de..bf4908084 100644
--- a/openviking_cli/client/base.py
+++ b/openviking_cli/client/base.py
@@ -253,7 +253,7 @@ class BaseClient(ABC):
@abstractmethod
async def find(
self,
- query: str,
+ query: str = "",
target_uri: Union[str, List[str]] = "",
limit: int = 10,
score_threshold: Optional[float] = None,
@@ -261,6 +261,7 @@ class BaseClient(ABC):
context_type: Optional[SearchContextTypeInput] = None,
tags: Optional[List[str]] = None,
telemetry: TelemetryRequest = False,
+ image: Optional[Any] = None,
) -> Any:
"""Semantic search without session context."""
...
@@ -268,7 +269,7 @@ class BaseClient(ABC):
@abstractmethod
async def search(
self,
- query: str,
+ query: str = "",
target_uri: Union[str, List[str]] = "",
session_id: Optional[str] = None,
limit: int = 10,
@@ -277,6 +278,7 @@ class BaseClient(ABC):
context_type: Optional[SearchContextTypeInput] = None,
tags: Optional[List[str]] = None,
telemetry: TelemetryRequest = False,
+ image: Optional[Any] = None,
) -> Any:
"""Semantic search with optional session context."""
...
diff --git a/openviking_cli/retrieve/types.py b/openviking_cli/retrieve/types.py
index b3540011f..8ee457d25 100644
--- a/openviking_cli/retrieve/types.py
+++ b/openviking_cli/retrieve/types.py
@@ -251,6 +251,8 @@ class TypedQuery:
intent: str
priority: int = 3
target_directories: List[str] = field(default_factory=list)
+ embedding_input: Optional[Any] = None
+ image_query: bool = False
@dataclass
diff --git a/sdk/go/README.md b/sdk/go/README.md
index ab5ef23ca..67d533811 100644
--- a/sdk/go/README.md
+++ b/sdk/go/README.md
@@ -85,6 +85,18 @@ for _, item := range results.Resources {
fmt.Println(item.URI, item.Score)
}
+// Search by image. Image accepts a local path, viking:// URI, HTTP URL, or data:image URI.
+imageResults, err := client.Find(ctx, "", &openviking.FindOptions{
+ TargetURI: "viking://resources/images",
+ Image: "./query.png",
+ Limit: 5,
+})
+similarPosters, err := client.Search(ctx, "similar poster", &openviking.SearchOptions{
+ TargetURI: "viking://resources/images",
+ Image: "viking://resources/images/poster.png",
+ Limit: 5,
+})
+
// Work with sessions.
session, err := client.CreateSession(ctx, &openviking.CreateSessionOptions{
SessionID: "demo-session",
@@ -96,7 +108,8 @@ commit, err := client.CommitSession(ctx, "demo-session", &openviking.CommitSessi
KeepRecentCount: 2,
})
-_, _, _ = resource, updated, session
+_, _, _, _ = resource, updated, imageResults, similarPosters
+_ = session
_ = commit
```
diff --git a/sdk/go/README_CN.md b/sdk/go/README_CN.md
index 9c94829fe..b668c4bb9 100644
--- a/sdk/go/README_CN.md
+++ b/sdk/go/README_CN.md
@@ -33,6 +33,34 @@ Go SDK 发送的身份请求头与 Python HTTP client 一致:
Go SDK 仅支持 HTTP 模式,不支持 Python embedded 模式,也不保留旧 `agent_id` 兼容路径。
+## 图片检索示例
+
+`FindOptions.Image` 和 `SearchOptions.Image` 支持本地路径、`viking://`、`http(s)://` 或 `data:image` URI;本地图片会在 SDK 内转成 base64 data URI,HTTP 请求体仍发送服务端字段 `image_url`。
+
+```go
+// 本地图片以图搜图
+imageResults, err := client.Find(ctx, "", &openviking.FindOptions{
+ TargetURI: "viking://resources/images",
+ Image: "./query.png",
+ Limit: 5,
+})
+
+// 已入库图片作为查询图
+storedImageResults, err := client.Find(ctx, "", &openviking.FindOptions{
+ TargetURI: "viking://resources/images",
+ Image: "viking://resources/images/cat.png",
+ Limit: 5,
+})
+
+// 图文联合检索
+similarPosters, err := client.Search(ctx, "红色海报风格", &openviking.SearchOptions{
+ TargetURI: "viking://resources/images",
+ Image: "./poster.png",
+ Limit: 5,
+})
+_, _, _ = imageResults, storedImageResults, similarPosters
+```
+
## 已实现接口
| 模块 | Go 方法 |
diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go
index d5ac01c75..2aac76607 100644
--- a/sdk/go/client_test.go
+++ b/sdk/go/client_test.go
@@ -165,6 +165,35 @@ func TestFindOmitsSearchFiltersWhenUnset(t *testing.T) {
}
}
+func TestFindSendsImageQuery(t *testing.T) {
+ imagePath := filepath.Join(t.TempDir(), "query.png")
+ if err := os.WriteFile(imagePath, []byte("\x89PNG\r\n\x1a\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ client, closeServer := testClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/api/v1/search/find" {
+ t.Fatalf("path = %s", r.URL.Path)
+ }
+ body := readJSONBody(t, r)
+ if got := body["query"]; got != "" {
+ t.Fatalf("query = %#v", got)
+ }
+ imageURL, ok := body["image_url"].(string)
+ if !ok || !strings.HasPrefix(imageURL, "data:image/png;base64,") {
+ t.Fatalf("image_url = %#v", body["image_url"])
+ }
+ writeOK(t, w, map[string]any{"resources": []any{}})
+ }))
+ defer closeServer()
+
+ if _, err := client.Find(context.Background(), "", &FindOptions{
+ TargetURI: "viking://resources/images",
+ Image: imagePath,
+ }); err != nil {
+ t.Fatal(err)
+ }
+}
+
func TestAdminCreatePathsAcceptInitialUserConfig(t *testing.T) {
var seen []map[string]any
client, closeServer := testClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -347,6 +376,27 @@ func TestSearchOmitsSearchFiltersWhenUnset(t *testing.T) {
}
}
+func TestSearchSendsImageURI(t *testing.T) {
+ client, closeServer := testClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/api/v1/search/search" {
+ t.Fatalf("path = %s", r.URL.Path)
+ }
+ body := readJSONBody(t, r)
+ if got := body["image_url"]; got != "viking://resources/images/cat.png" {
+ t.Fatalf("image_url = %#v", got)
+ }
+ writeOK(t, w, map[string]any{"resources": []any{}})
+ }))
+ defer closeServer()
+
+ if _, err := client.Search(context.Background(), "similar poster", &SearchOptions{
+ TargetURI: "viking://resources/images",
+ Image: "viking://resources/images/cat.png",
+ }); err != nil {
+ t.Fatal(err)
+ }
+}
+
func TestErrorEnvelopePreservesCodeDetailsAndStatus(t *testing.T) {
client, closeServer := testClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
writeAPIError(t, w, http.StatusNotFound, "NOT_FOUND", map[string]any{"resource": "viking://resources/missing"})
diff --git a/sdk/go/helpers.go b/sdk/go/helpers.go
index cf2927c7f..e74976589 100644
--- a/sdk/go/helpers.go
+++ b/sdk/go/helpers.go
@@ -1,9 +1,14 @@
package openviking
import (
+ "encoding/base64"
"fmt"
+ "mime"
"net/url"
+ "os"
+ "path/filepath"
"strconv"
+ "strings"
)
func setString(m map[string]any, key, value string) {
@@ -105,3 +110,31 @@ func normalizeTarget(target any) any {
return v
}
}
+
+func normalizeImageInput(image string) (string, error) {
+ if image == "" || strings.HasPrefix(image, "data:image/") ||
+ strings.HasPrefix(image, "http://") ||
+ strings.HasPrefix(image, "https://") ||
+ strings.HasPrefix(image, "viking://") {
+ return image, nil
+ }
+ info, err := os.Stat(image)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return image, nil
+ }
+ return "", err
+ }
+ if info.IsDir() {
+ return image, nil
+ }
+ data, err := os.ReadFile(image)
+ if err != nil {
+ return "", err
+ }
+ mimeType := mime.TypeByExtension(strings.ToLower(filepath.Ext(image)))
+ if mimeType == "" || !strings.HasPrefix(mimeType, "image/") {
+ mimeType = "image/png"
+ }
+ return fmt.Sprintf("data:%s;base64,%s", mimeType, base64.StdEncoding.EncodeToString(data)), nil
+}
diff --git a/sdk/go/retrieval.go b/sdk/go/retrieval.go
index cdded2cc4..c8254a518 100644
--- a/sdk/go/retrieval.go
+++ b/sdk/go/retrieval.go
@@ -18,11 +18,16 @@ func (c *Client) Find(ctx context.Context, queryText string, opts *FindOptions)
if opts.NodeLimit != nil {
actualLimit = *opts.NodeLimit
}
+ imageURL, err := normalizeImageInput(opts.Image)
+ if err != nil {
+ return nil, err
+ }
payload := map[string]any{
"query": queryText,
"target_uri": normalizeTarget(opts.TargetURI),
"limit": actualLimit,
}
+ setString(payload, "image_url", imageURL)
setAny(payload, "score_threshold", opts.ScoreThreshold)
setAny(payload, "filter", opts.Filter)
setAny(payload, "context_type", opts.ContextType)
@@ -34,7 +39,7 @@ func (c *Client) Find(ctx context.Context, queryText string, opts *FindOptions)
}
setAny(payload, "telemetry", opts.Telemetry)
var result FindResult
- err := c.doJSON(ctx, http.MethodPost, "/api/v1/search/find", nil, payload, &result)
+ err = c.doJSON(ctx, http.MethodPost, "/api/v1/search/find", nil, payload, &result)
return &result, err
}
@@ -51,11 +56,16 @@ func (c *Client) Search(ctx context.Context, queryText string, opts *SearchOptio
if opts.NodeLimit != nil {
actualLimit = *opts.NodeLimit
}
+ imageURL, err := normalizeImageInput(opts.Image)
+ if err != nil {
+ return nil, err
+ }
payload := map[string]any{
"query": queryText,
"target_uri": normalizeTarget(opts.TargetURI),
"limit": actualLimit,
}
+ setString(payload, "image_url", imageURL)
setString(payload, "session_id", opts.SessionID)
setAny(payload, "score_threshold", opts.ScoreThreshold)
setAny(payload, "filter", opts.Filter)
@@ -68,7 +78,7 @@ func (c *Client) Search(ctx context.Context, queryText string, opts *SearchOptio
}
setAny(payload, "telemetry", opts.Telemetry)
var result FindResult
- err := c.doJSON(ctx, http.MethodPost, "/api/v1/search/search", nil, payload, &result)
+ err = c.doJSON(ctx, http.MethodPost, "/api/v1/search/search", nil, payload, &result)
return &result, err
}
diff --git a/sdk/go/types.go b/sdk/go/types.go
index d6d078ce4..d06964c17 100644
--- a/sdk/go/types.go
+++ b/sdk/go/types.go
@@ -189,6 +189,7 @@ type ReindexOptions struct {
// FindOptions controls Find.
type FindOptions struct {
TargetURI any
+ Image string
Limit int
NodeLimit *int
ScoreThreshold *float64
@@ -204,6 +205,7 @@ type FindOptions struct {
// SearchOptions controls Search.
type SearchOptions struct {
TargetURI any
+ Image string
SessionID string
Limit int
NodeLimit *int
diff --git a/sdk/python/README.md b/sdk/python/README.md
index a80ed32dd..17f4d2401 100644
--- a/sdk/python/README.md
+++ b/sdk/python/README.md
@@ -182,6 +182,13 @@ result = client.find("hello", limit=5)
print(result)
```
+Image search uses the same methods. Pass a local path, bytes, data URI, HTTP URL, or `viking://` URI with `image`. The server must use a multimodal embedding model.
+
+```python
+result = client.find(image="/path/to/photo.png", limit=5)
+result = client.search("similar poster", image="viking://resources/poster.png")
+```
+
## Admin Operations
If you connect with a root key, the SDK also exposes admin APIs such as:
diff --git a/sdk/python/README_CN.md b/sdk/python/README_CN.md
index dd1753081..e4404528b 100644
--- a/sdk/python/README_CN.md
+++ b/sdk/python/README_CN.md
@@ -182,6 +182,13 @@ result = client.find("hello", limit=5)
print(result)
```
+图片搜索也使用同一组方法。`image` 支持本地路径、bytes、data URI、HTTP URL 或 `viking://` URI;服务端需要使用 multimodal embedding 模型。
+
+```python
+result = client.find(image="/path/to/photo.png", limit=5)
+result = client.search("similar poster", image="viking://resources/poster.png")
+```
+
## 管理员操作
如果你使用 root key 连接,SDK 也暴露了管理员 API,例如:
diff --git a/sdk/python/openviking_sdk/client.py b/sdk/python/openviking_sdk/client.py
index 0aa42c560..af5066493 100644
--- a/sdk/python/openviking_sdk/client.py
+++ b/sdk/python/openviking_sdk/client.py
@@ -1,6 +1,9 @@
from __future__ import annotations
+import base64
import inspect
+import mimetypes
+import os
import tempfile
import uuid
import zipfile
@@ -59,6 +62,32 @@ ERROR_CODE_TO_EXCEPTION = {
}
+def _image_mime_type(file_name: str = "") -> str:
+ mime_type, _ = mimetypes.guess_type(file_name or "")
+ if mime_type and mime_type.startswith("image/"):
+ return mime_type
+ return "image/png"
+
+
+def _image_to_data_uri(data: bytes | bytearray | memoryview, file_name: str = "") -> str:
+ encoded = base64.b64encode(bytes(data)).decode("ascii")
+ return f"data:{_image_mime_type(file_name)};base64,{encoded}"
+
+
+def _normalize_image_input(image: Any) -> Optional[str]:
+ if image is None:
+ return None
+ if isinstance(image, (bytes, bytearray, memoryview)):
+ return _image_to_data_uri(image)
+ value = os.fspath(image) if isinstance(image, os.PathLike) else str(image)
+ if value.startswith(("data:image/", "http://", "https://", "viking://")):
+ return value
+ path = Path(value).expanduser()
+ if path.is_file():
+ return _image_to_data_uri(path.read_bytes(), path.name)
+ return value
+
+
class VikingURI:
@staticmethod
def normalize(uri: str) -> str:
@@ -877,7 +906,7 @@ class AsyncHTTPClient:
async def find(
self,
- query: str,
+ query: str = "",
target_uri: Union[str, List[str]] = "",
limit: int = 10,
node_limit: Optional[int] = None,
@@ -886,10 +915,12 @@ class AsyncHTTPClient:
context_type: Optional[Any] = None,
tags: Optional[List[str]] = None,
telemetry: Any = False,
+ image: Any = None,
) -> Dict[str, Any]:
actual_limit = node_limit if node_limit is not None else limit
payload = {
"query": query,
+ "image_url": _normalize_image_input(image),
"target_uri": self._normalize_target_uri(target_uri),
"limit": actual_limit,
"score_threshold": score_threshold,
@@ -904,7 +935,7 @@ class AsyncHTTPClient:
async def search(
self,
- query: str,
+ query: str = "",
target_uri: Union[str, List[str]] = "",
session: Optional[Any] = None,
session_id: Optional[str] = None,
@@ -915,11 +946,13 @@ class AsyncHTTPClient:
context_type: Optional[Any] = None,
tags: Optional[List[str]] = None,
telemetry: Any = False,
+ image: Any = None,
) -> Dict[str, Any]:
actual_limit = node_limit if node_limit is not None else limit
sid = session_id or (session.session_id if session else None)
payload = {
"query": query,
+ "image_url": _normalize_image_input(image),
"target_uri": self._normalize_target_uri(target_uri),
"session_id": sid,
"limit": actual_limit,
@@ -1791,7 +1824,7 @@ class SyncHTTPClient:
def find(
self,
- query: str,
+ query: str = "",
target_uri: Union[str, List[str]] = "",
limit: int = 10,
node_limit: Optional[int] = None,
@@ -1800,6 +1833,7 @@ class SyncHTTPClient:
context_type: Optional[Any] = None,
tags: Optional[List[str]] = None,
telemetry: Any = False,
+ image: Any = None,
) -> Dict[str, Any]:
return run_async(
self._async_client.find(
@@ -1812,12 +1846,13 @@ class SyncHTTPClient:
context_type=context_type,
tags=tags,
telemetry=telemetry,
+ image=image,
)
)
def search(
self,
- query: str,
+ query: str = "",
target_uri: Union[str, List[str]] = "",
session: Optional[Any] = None,
session_id: Optional[str] = None,
@@ -1828,6 +1863,7 @@ class SyncHTTPClient:
context_type: Optional[Any] = None,
tags: Optional[List[str]] = None,
telemetry: Any = False,
+ image: Any = None,
) -> Dict[str, Any]:
actual_session_id = session_id
if actual_session_id is None and session is not None:
@@ -1844,6 +1880,7 @@ class SyncHTTPClient:
context_type=context_type,
tags=tags,
telemetry=telemetry,
+ image=image,
)
)
diff --git a/tests/misc/test_vikingfs_find_without_rerank.py b/tests/misc/test_vikingfs_find_without_rerank.py
index 17880c668..c250aaa46 100644
--- a/tests/misc/test_vikingfs_find_without_rerank.py
+++ b/tests/misc/test_vikingfs_find_without_rerank.py
@@ -101,3 +101,54 @@ async def test_find_works_without_rerank_config(monkeypatch) -> None:
assert captured["scope_dsl"] == {"category": "doc"}
assert captured["level"] is None
fs._ensure_access.assert_called_once_with("viking://resources/docs", request_ctx)
+
+
+async def test_find_accepts_image_url_without_text_query(monkeypatch) -> None:
+ fs = _make_viking_fs()
+ captured = {}
+
+ class FakeRetriever:
+ def __init__(self, storage, embedder, rerank_config, retrieval_config):
+ pass
+
+ async def retrieve(
+ self,
+ typed_query,
+ ctx,
+ limit,
+ score_threshold,
+ scope_dsl,
+ level,
+ ):
+ captured["typed_query"] = typed_query
+ return QueryResult(
+ query=typed_query,
+ matched_contexts=[
+ MatchedContext(
+ uri="viking://resources/photos/cat.png",
+ context_type=ContextType.RESOURCE,
+ score=0.9,
+ )
+ ],
+ searched_directories=["viking://resources"],
+ )
+
+ monkeypatch.setattr(
+ "openviking.retrieve.hierarchical_retriever.HierarchicalRetriever",
+ FakeRetriever,
+ )
+
+ result = await fs.find(
+ "",
+ target_uri="viking://resources",
+ image_url="data:image/png;base64,abc",
+ )
+
+ assert result.total == 1
+ typed_query = captured["typed_query"]
+ assert typed_query.query == ""
+ assert typed_query.image_query is True
+ assert typed_query.embedding_input == [
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}}
+ ]
diff --git a/tests/retrieve/test_hierarchical_retriever_image.py b/tests/retrieve/test_hierarchical_retriever_image.py
new file mode 100644
index 000000000..5dd5e2547
--- /dev/null
+++ b/tests/retrieve/test_hierarchical_retriever_image.py
@@ -0,0 +1,118 @@
+import pytest
+
+from openviking.models.embedder.base import EmbedResult
+from openviking.retrieve.hierarchical_retriever import HierarchicalRetriever
+from openviking.server.identity import RequestContext, Role
+from openviking_cli.exceptions import InvalidArgumentError
+from openviking_cli.retrieve.types import TypedQuery
+from openviking_cli.session.user_id import UserIdentifier
+
+
+class FakeProxy:
+ captured = {}
+
+ def __init__(self, _storage, _ctx):
+ pass
+
+ @property
+ def collection_name(self):
+ return "test"
+
+ async def collection_exists_bound(self):
+ return True
+
+ async def search_in_tenant(self, **kwargs):
+ self.captured.update(kwargs)
+ return [
+ {
+ "uri": "viking://resources/photos/cat.png",
+ "context_type": "resource",
+ "level": 2,
+ "_score": 0.9,
+ "abstract": "cat",
+ },
+ {
+ "uri": "viking://resources/docs/cat.md",
+ "context_type": "resource",
+ "level": 2,
+ "_score": 0.8,
+ "abstract": "cat doc",
+ },
+ ]
+
+
+class MultimodalEmbedder:
+ supports_multimodal = True
+
+ def __init__(self):
+ self.seen = None
+
+ def prepare_embedding_input(self, content):
+ self.seen = content
+ return content
+
+ async def embed_async(self, content, is_query=False):
+ return EmbedResult(dense_vector=[1.0])
+
+
+class TextOnlyEmbedder(MultimodalEmbedder):
+ supports_multimodal = False
+
+
+def _ctx():
+ return RequestContext(user=UserIdentifier("acc", "user"), role=Role.USER)
+
+
+async def test_image_query_uses_multimodal_input_without_filtering_non_images(monkeypatch):
+ monkeypatch.setattr(
+ "openviking.retrieve.hierarchical_retriever.VikingDBManagerProxy",
+ FakeProxy,
+ )
+ embedder = MultimodalEmbedder()
+ retriever = HierarchicalRetriever(storage=object(), embedder=embedder)
+ query_input = [{"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}}]
+
+ result = await retriever.retrieve(
+ TypedQuery(
+ query="",
+ context_type=None,
+ intent="",
+ embedding_input=query_input,
+ image_query=True,
+ ),
+ ctx=_ctx(),
+ limit=2,
+ )
+
+ assert embedder.seen == query_input
+ assert [ctx.uri for ctx in result.matched_contexts] == [
+ "viking://resources/photos/cat.png",
+ "viking://resources/docs/cat.md",
+ ]
+ assert FakeProxy.captured["context_type"] == "resource"
+ assert FakeProxy.captured["level"] == [2]
+ assert FakeProxy.captured["limit"] == 50
+
+
+async def test_image_query_requires_multimodal_embedder(monkeypatch):
+ monkeypatch.setattr(
+ "openviking.retrieve.hierarchical_retriever.VikingDBManagerProxy",
+ FakeProxy,
+ )
+ retriever = HierarchicalRetriever(storage=object(), embedder=TextOnlyEmbedder())
+
+ with pytest.raises(InvalidArgumentError, match="multimodal embedding"):
+ await retriever.retrieve(
+ TypedQuery(
+ query="",
+ context_type=None,
+ intent="",
+ embedding_input=[
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}}
+ ],
+ image_query=True,
+ ),
+ ctx=_ctx(),
+ )
diff --git a/tests/service/test_search_service_image.py b/tests/service/test_search_service_image.py
new file mode 100644
index 000000000..50bead176
--- /dev/null
+++ b/tests/service/test_search_service_image.py
@@ -0,0 +1,34 @@
+import pytest
+
+from openviking.server.identity import RequestContext, Role
+from openviking.service.search_service import SearchService
+from openviking_cli.session.user_id import UserIdentifier
+
+
+class FakeVikingFS:
+ def __init__(self):
+ self.image_url = None
+
+ async def read_file_bytes(self, uri, ctx=None):
+ assert uri == "viking://resources/cat.png"
+ return b"\x89PNG\r\n\x1a\n"
+
+ async def find(self, **kwargs):
+ self.image_url = kwargs["image_url"]
+ return {"ok": True}
+
+
+async def test_search_service_resolves_viking_image_url_to_data_uri():
+ fs = FakeVikingFS()
+ service = SearchService(fs)
+ ctx = RequestContext(user=UserIdentifier("acc", "user"), role=Role.USER)
+
+ result = await service.find(
+ query="",
+ image_url="viking://resources/cat.png",
+ ctx=ctx,
+ )
+
+ assert result == {"ok": True}
+ assert fs.image_url.startswith("data:image/png;base64,")
diff --git a/tests/unit/test_image_search.py b/tests/unit/test_image_search.py
new file mode 100644
index 000000000..070cdcad1
--- /dev/null
+++ b/tests/unit/test_image_search.py
@@ -0,0 +1,38 @@
+from openviking.models.embedder.base import DenseEmbedderBase, EmbedResult
+from openviking.utils.image_search import (
+ build_multimodal_embedding_input,
+ image_bytes_to_data_uri,
+ normalize_client_image_input,
+)
+
+
+class TextOnlyEmbedder(DenseEmbedderBase):
+ def embed(self, content, is_query=False):
+ return EmbedResult(dense_vector=[1.0])
+
+ def get_dimension(self) -> int:
+ return 1
+
+
+class MultimodalEmbedder(TextOnlyEmbedder):
+ @property
+ def supports_multimodal(self) -> bool:
+ return True
+
+
+def test_text_only_embedder_drops_image_parts():
+ parts = build_multimodal_embedding_input(
+ "cat photo",
+ "data:image/png;base64,abc",
+ )
+
+ assert TextOnlyEmbedder("test").prepare_embedding_input(parts) == "cat photo"
+ assert MultimodalEmbedder("test").prepare_embedding_input(parts) == parts
+
+
+def test_image_helpers_accept_base64_bytes_and_image_uris(tmp_path):
+ image_path = tmp_path / "sample.jpg"
+ image_path.write_bytes(b"jpg")
+
+ assert image_bytes_to_data_uri(b"png").startswith("data:image/png;base64,")
+ assert normalize_client_image_input(image_path).startswith("data:image/jpeg;base64,")
diff --git a/tests/unit/test_vectorize_file_strategy.py b/tests/unit/test_vectorize_file_strategy.py
index 2bd4237f7..c702d8184 100644
--- a/tests/unit/test_vectorize_file_strategy.py
+++ b/tests/unit/test_vectorize_file_strategy.py
@@ -285,6 +285,66 @@ async def test_vectorize_text_summary_first_reuses_single_file_read(monkeypatch)
assert fs.read_file_bytes_calls == 0
+async def test_vectorize_image_file_enqueues_summary_and_image(monkeypatch):
+ queue = DummyQueue()
+ fs = DummyFS(b"\x89PNG\r\n\x1a\nimage")
+ monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue))
+ monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: fs)
+ monkeypatch.setattr(
+ embedding_utils,
+ "get_openviking_config",
+ lambda: types.SimpleNamespace(
+ embedding=types.SimpleNamespace(text_source="summary_first", max_input_tokens=1000)
+ ),
+ )
+
+ await embedding_utils.vectorize_file(
+ file_path="viking://user/default/resources/photo.png",
+ summary_dict={"name": "photo.png", "summary": "a cat on a sofa"},
+ parent_uri="viking://user/default/resources",
+ ctx=DummyReq(),
+ )
+
+ assert len(queue.items) == 1
+ msg = queue.items[0]
+ assert msg.message[0] == {"type": "text", "text": "a cat on a sofa"}
+ assert msg.message[1]["type"] == "image_url"
+ assert msg.message[1]["image_url"]["url"].startswith("data:image/png;base64,")
+ assert msg.context_data["content"] == "a cat on a sofa"
+
+
+async def test_vectorize_image_file_falls_back_to_summary_when_image_unreadable(monkeypatch):
+ class UnreadableImageFS(DummyFS):
+ async def read_file_bytes(self, _path, ctx=None):
+ self.read_file_bytes_calls += 1
+ raise OSError("cannot read")
+
+ queue = DummyQueue()
+ fs = UnreadableImageFS("")
+ monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue))
+ monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: fs)
+ monkeypatch.setattr(
+ embedding_utils,
+ "get_openviking_config",
+ lambda: types.SimpleNamespace(
+ embedding=types.SimpleNamespace(text_source="summary_first", max_input_tokens=1000)
+ ),
+ )
+
+ await embedding_utils.vectorize_file(
+ file_path="viking://user/default/resources/photo.png",
+ summary_dict={"name": "photo.png", "summary": "fallback summary"},
+ parent_uri="viking://user/default/resources",
+ ctx=DummyReq(),
+ )
+
+ assert len(queue.items) == 1
+ assert queue.items[0].message == "fallback summary"
+ assert fs.read_file_bytes_calls == 1
+
+
@pytest.mark.asyncio
async def test_vectorize_text_file_reuses_summary_content_without_reread(monkeypatch):
queue = DummyQueue()