Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(media): downsample oversized image model inputs (#3965)

MaojiaSheng5aed7f72b4b1
diff --git a/openviking/parse/parsers/media/large_image_processor.py b/openviking/parse/parsers/media/large_image_processor.py
index 6b2bde001..9af149a12 100644
--- a/openviking/parse/parsers/media/large_image_processor.py
+++ b/openviking/parse/parsers/media/large_image_processor.py
@@ -18,6 +18,12 @@ from typing import List, Optional, Tuple
from PIL import Image, ImageDraw, ImageFont
+from openviking.utils.media_limits import (
+ DEFAULT_IMAGE_MAX_TILE_DIMENSION_PX,
+ DEFAULT_IMAGE_PREVIEW_MAX_DIMENSION,
+ DEFAULT_IMAGE_TILE_OVERLAP_PX,
+ is_large_image_by_size,
+)
from openviking_cli.utils.config.parser_config import ImageConfig
from openviking_cli.utils.logger import get_logger
@@ -28,21 +34,10 @@ logger = get_logger(__name__)
# Configuration
# =============================================================================
-# Thresholds for triggering large image processing (fallback defaults when no config)
-MAX_FILE_SIZE_MB = 10.0 # 10 MB
-LARGE_IMAGE_THRESHOLD_DIMENSION = 4096 # 4096 pixels
-
-# Target limits for individual tiles
-MAX_TILE_DIMENSION_PX = 2048 # 2048 pixels
-TILE_OVERLAP_PX = 2 # 2 pixels of overlap on each side
-
# Quality settings for JPEG compression
PREVIEW_QUALITY_START = 85
TILE_QUALITY = 90
-# Preview dimension fallback (when no config provides preview_max_dimension)
-PREVIEW_MAX_DIMENSION = 2048
-
@dataclass
class TileInfo:
@@ -108,11 +103,12 @@ def needs_large_image_processing(
Returns:
True if large image processing is needed
"""
- max_file_size_mb = config.max_file_size_mb if config else MAX_FILE_SIZE_MB
- max_dimension_px = config.large_image_threshold_dimension if config else LARGE_IMAGE_THRESHOLD_DIMENSION
-
- file_size_mb = get_image_size_mb(file_path)
- return file_size_mb > max_file_size_mb or width > max_dimension_px or height > max_dimension_px
+ return is_large_image_by_size(
+ file_size_bytes=file_path.stat().st_size,
+ width=width,
+ height=height,
+ config=config,
+ )
def create_low_res_preview(
@@ -131,7 +127,7 @@ def create_low_res_preview(
Returns:
Preview image bytes in JPEG format
"""
- max_dimension_px = config.preview_max_dimension if config else PREVIEW_MAX_DIMENSION
+ max_dimension_px = config.preview_max_dimension if config else DEFAULT_IMAGE_PREVIEW_MAX_DIMENSION
# Work on a copy
img = img.copy()
@@ -171,8 +167,10 @@ def calculate_grid_dimensions(
Returns:
Tuple of (num_rows, num_cols)
"""
- max_tile_dimension_px = config.max_tile_dimension_px if config else MAX_TILE_DIMENSION_PX
- tile_overlap_px = config.tile_overlap_px if config else TILE_OVERLAP_PX
+ max_tile_dimension_px = (
+ config.max_tile_dimension_px if config else DEFAULT_IMAGE_MAX_TILE_DIMENSION_PX
+ )
+ tile_overlap_px = config.tile_overlap_px if config else DEFAULT_IMAGE_TILE_OVERLAP_PX
effective_tile = max_tile_dimension_px - tile_overlap_px * 2
if effective_tile <= 0:
@@ -204,7 +202,7 @@ def calculate_tile_positions(
Returns:
List of (x1, y1, x2, y2) tuples
"""
- tile_overlap_px = config.tile_overlap_px if config else TILE_OVERLAP_PX
+ tile_overlap_px = config.tile_overlap_px if config else DEFAULT_IMAGE_TILE_OVERLAP_PX
# Calculate base tile size without overlap
base_tile_width = ceil(width / cols)
@@ -353,8 +351,9 @@ def create_grid_overlay(
Returns:
Grid overlay image bytes
"""
- tile_overlap_px = config.tile_overlap_px if config else TILE_OVERLAP_PX
- preview_max_dimension = config.preview_max_dimension if config else PREVIEW_MAX_DIMENSION
+ preview_max_dimension = (
+ config.preview_max_dimension if config else DEFAULT_IMAGE_PREVIEW_MAX_DIMENSION
+ )
# Resize to a reasonable size for overlay drawing
ow, oh = img.size
@@ -442,9 +441,6 @@ def process_large_image(
Returns:
LargeImageResult with processing results
"""
- # Use config values or defaults
- max_file_size_mb = config.max_file_size_mb if config else MAX_FILE_SIZE_MB
-
# Load image if not provided
own_img = False
if img is None:
diff --git a/openviking/parse/parsers/media/utils.py b/openviking/parse/parsers/media/utils.py
index daeb471eb..7ce8e5306 100644
--- a/openviking/parse/parsers/media/utils.py
+++ b/openviking/parse/parsers/media/utils.py
@@ -12,6 +12,7 @@ from openviking.core.path_variables import CalendarVariableProvider
from openviking.parse.parsers.constants import TYPESCRIPT_MPEG_TS_EXTENSION
from openviking.prompts import render_prompt
from openviking.storage.viking_fs import get_viking_fs
+from openviking.utils.image_search import prepare_image_bytes_for_model
from openviking.utils.media_limits import MAX_MEDIA_FILE_BYTES
from openviking_cli.utils.config import get_openviking_config
from openviking_cli.utils.logger import get_logger
@@ -264,7 +265,8 @@ async def generate_image_summary(
Dictionary with "name" and "summary" keys
"""
viking_fs = get_viking_fs()
- vlm = get_openviking_config().vlm
+ config = get_openviking_config()
+ vlm = config.vlm
file_name = original_filename
try:
@@ -295,7 +297,12 @@ async def generate_image_summary(
async with llm_sem or asyncio.Semaphore(1):
response = await vlm.get_vision_completion_async(
prompt=prompt,
- images=[image_bytes],
+ images=[
+ prepare_image_bytes_for_model(
+ image_bytes,
+ config=getattr(config, "image", None),
+ )
+ ],
)
logger.info(
diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.py
index a34b463d3..24b193030 100644
--- a/openviking/utils/embedding_utils.py
+++ b/openviking/utils/embedding_utils.py
@@ -31,7 +31,7 @@ from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConve
from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs
from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.utils.embedding_input import truncate_embedding_input
-from openviking.utils.image_search import image_bytes_to_data_uri
+from openviking.utils.image_search import image_bytes_to_model_data_uri
from openviking.utils.ingest_options import IngestOptions
from openviking.utils.time_utils import parse_iso_datetime
from openviking_cli.utils import VikingURI, get_logger
@@ -272,11 +272,14 @@ async def _build_image_data_uri(
) -> Optional[str]:
"""Read an image file and encode it as a base64 ``data:`` URI.
+ Oversized images are downsampled only for the embedding request. The
+ original resource bytes in VikingFS are left unchanged.
Returns None if the image cannot be read.
"""
try:
content = await viking_fs.read_file_bytes(file_path, ctx=ctx)
- return image_bytes_to_data_uri(content, file_name)
+ image_config = getattr(get_openviking_config(), "image", None)
+ return image_bytes_to_model_data_uri(content, file_name, config=image_config)
except Exception as e:
logger.warning(f"Failed to read image for multimodal vectorization {file_path}: {e}")
return None
@@ -571,7 +574,9 @@ async def vectorize_file(
if summary:
context.set_vectorize(Vectorize(text=summary))
else:
- logger.warning(f"No summary available for {file_path}, skipping vectorization")
+ logger.warning(
+ f"No summary available for {file_path}, skipping vectorization"
+ )
return False
else:
embedding_text = truncate_embedding_input(
diff --git a/openviking/utils/image_search.py b/openviking/utils/image_search.py
index e44eb279c..d5a8b7d1f 100644
--- a/openviking/utils/image_search.py
+++ b/openviking/utils/image_search.py
@@ -5,11 +5,17 @@
from __future__ import annotations
import base64
+import io
import mimetypes
import os
from pathlib import Path
from typing import Any, Dict, List, Optional
+from PIL import Image, UnidentifiedImageError
+
+from openviking.utils.media_limits import is_large_image_by_size
+from openviking_cli.utils.config.parser_config import ImageConfig
+
def image_mime_type(file_name: str = "") -> str:
mime_type, _ = mimetypes.guess_type(file_name or "")
@@ -23,6 +29,57 @@ def image_bytes_to_data_uri(data: bytes | bytearray | memoryview, file_name: str
return f"data:{image_mime_type(file_name)};base64,{encoded}"
+def _prepare_image_bytes_for_model(
+ data: bytes | bytearray | memoryview,
+ config: ImageConfig | None = None,
+) -> tuple[bytes, bool]:
+ """Return model-ready image bytes and whether they differ from the input."""
+ content = bytes(data)
+ try:
+ with Image.open(io.BytesIO(content)) as img:
+ width, height = img.size
+ image_config = config or ImageConfig()
+ if not is_large_image_by_size(
+ file_size_bytes=len(content),
+ width=width,
+ height=height,
+ config=image_config,
+ ):
+ return content, False
+
+ preview = img.convert("RGB")
+
+ max_dimension = image_config.preview_max_dimension
+ if width > max_dimension or height > max_dimension:
+ ratio = min(max_dimension / width, max_dimension / height)
+ new_size = (max(1, int(width * ratio)), max(1, int(height * ratio)))
+ preview = preview.resize(new_size, Image.Resampling.LANCZOS)
+
+ buf = io.BytesIO()
+ preview.save(buf, format="JPEG", quality=85, optimize=True)
+ return buf.getvalue(), True
+ except (UnidentifiedImageError, OSError, ValueError):
+ return content, False
+
+
+def prepare_image_bytes_for_model(
+ data: bytes | bytearray | memoryview,
+ config: ImageConfig | None = None,
+) -> bytes:
+ """Downsample oversized image bytes for model requests without changing storage."""
+ model_bytes, _ = _prepare_image_bytes_for_model(data, config=config)
+ return model_bytes
+
+
+def image_bytes_to_model_data_uri(
+ data: bytes | bytearray | memoryview,
+ file_name: str = "",
+ config: ImageConfig | None = None,
+) -> str:
+ model_bytes, changed = _prepare_image_bytes_for_model(data, config=config)
+ return image_bytes_to_data_uri(model_bytes, "model_input.jpg" if changed else file_name)
+
+
def is_data_image_uri(value: str) -> bool:
return value.startswith("data:image/") and ";base64," in value
diff --git a/openviking/utils/media_limits.py b/openviking/utils/media_limits.py
index d96038730..c64fbba3d 100644
--- a/openviking/utils/media_limits.py
+++ b/openviking/utils/media_limits.py
@@ -1,5 +1,37 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
-"""Shared resource limits for audio and video processing."""
+"""Shared media limits and limit checks."""
+
+from typing import Protocol
MAX_MEDIA_FILE_BYTES = 512 * 1024 * 1024
+
+DEFAULT_LARGE_IMAGE_MAX_FILE_SIZE_MB = 10.0
+DEFAULT_LARGE_IMAGE_THRESHOLD_DIMENSION = 4096
+DEFAULT_IMAGE_PREVIEW_MAX_DIMENSION = 2048
+DEFAULT_IMAGE_MAX_TILE_DIMENSION_PX = 2048
+DEFAULT_IMAGE_TILE_OVERLAP_PX = 2
+
+
+class ImageLimitConfig(Protocol):
+ max_file_size_mb: float
+ large_image_threshold_dimension: int
+
+
+def is_large_image_by_size(
+ *,
+ file_size_bytes: int,
+ width: int,
+ height: int,
+ config: ImageLimitConfig | None = None,
+) -> bool:
+ """Return whether image bytes exceed the configured large-image limits."""
+ max_file_size_mb = config.max_file_size_mb if config else DEFAULT_LARGE_IMAGE_MAX_FILE_SIZE_MB
+ max_dimension_px = (
+ config.large_image_threshold_dimension
+ if config
+ else DEFAULT_LARGE_IMAGE_THRESHOLD_DIMENSION
+ )
+
+ file_size_mb = file_size_bytes / (1024 * 1024)
+ return file_size_mb > max_file_size_mb or width > max_dimension_px or height > max_dimension_px
diff --git a/tests/parse/test_media_understanding_summary.py b/tests/parse/test_media_understanding_summary.py
index 1ebe34cec..7d5fdbce6 100644
--- a/tests/parse/test_media_understanding_summary.py
+++ b/tests/parse/test_media_understanding_summary.py
@@ -1,11 +1,14 @@
import asyncio
+import io
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
+from PIL import Image
from openviking.parse.parsers.media import utils as media_utils
+from openviking_cli.utils.config.parser_config import ImageConfig
from openviking_cli.utils.config.vlm_config import VLMConfig
@@ -24,6 +27,9 @@ class _FS:
return b""
return self.content[offset : offset + size]
+ async def read_file_bytes(self, _uri, ctx=None):
+ return self.content
+
class _BlockingReadFS:
def __init__(self):
@@ -102,6 +108,15 @@ class _CapturingPathClient(_MediaVLM):
return "# Clip\n\nUseful summary."
+class _ImageVLM:
+ def __init__(self):
+ self.images = []
+
+ async def get_vision_completion_async(self, *, prompt, images):
+ self.images = images
+ return "image summary"
+
+
def _config(model_config, *, max_chars=4000):
if hasattr(model_config, "get_client_instance"):
vlm = model_config.get_client_instance()
@@ -118,6 +133,17 @@ def _config(model_config, *, max_chars=4000):
)
+def _jpeg_bytes(width: int, height: int) -> bytes:
+ buf = io.BytesIO()
+ Image.new("RGB", (width, height), "white").save(buf, format="JPEG")
+ return buf.getvalue()
+
+
+def _image_size(data: bytes) -> tuple[int, int]:
+ with Image.open(io.BytesIO(data)) as img:
+ return img.size
+
+
def _lazy_client(*, return_value=None, side_effect=None):
async def invoke(prepare_media=None, **_kwargs):
if prepare_media is not None:
@@ -176,6 +202,32 @@ async def test_media_concurrency_bounds_staging_and_inference(monkeypatch):
assert fs.peak_reads == 2
+async def test_image_summary_downsamples_large_model_input(monkeypatch):
+ original = _jpeg_bytes(80, 220)
+ fs = _FS(original)
+ vlm = _ImageVLM()
+ config = SimpleNamespace(
+ vlm=vlm,
+ image=ImageConfig(
+ preview_max_dimension=64,
+ max_file_size_mb=100.0,
+ large_image_threshold_dimension=100,
+ ),
+ )
+ monkeypatch.setattr(media_utils, "get_viking_fs", lambda: fs)
+ monkeypatch.setattr(media_utils, "get_openviking_config", lambda: config)
+
+ result = await media_utils.generate_image_summary(
+ "viking://resources/docs/large.jpg",
+ "large.jpg",
+ )
+
+ assert result == {"name": "large.jpg", "summary": "image summary"}
+ assert len(vlm.images) == 1
+ assert max(_image_size(vlm.images[0])) <= 64
+ assert fs.content == original
+
+
async def test_unknown_size_media_stops_at_hard_staging_limit(
monkeypatch,
):
diff --git a/tests/unit/test_image_search.py b/tests/unit/test_image_search.py
index 070cdcad1..ddcf4d7d5 100644
--- a/tests/unit/test_image_search.py
+++ b/tests/unit/test_image_search.py
@@ -1,9 +1,16 @@
+import base64
+import io
+
+from PIL import Image
+
from openviking.models.embedder.base import DenseEmbedderBase, EmbedResult
from openviking.utils.image_search import (
build_multimodal_embedding_input,
image_bytes_to_data_uri,
+ image_bytes_to_model_data_uri,
normalize_client_image_input,
)
+from openviking_cli.utils.config.parser_config import ImageConfig
class TextOnlyEmbedder(DenseEmbedderBase):
@@ -36,3 +43,24 @@ def test_image_helpers_accept_base64_bytes_and_image_uris(tmp_path):
assert image_bytes_to_data_uri(b"png").startswith("data:image/png;base64,")
assert normalize_client_image_input(image_path).startswith("data:image/jpeg;base64,")
+
+
+def test_model_image_input_downsamples_jpeg_incompatible_modes():
+ buf = io.BytesIO()
+ Image.new("LA", (80, 220), color=(255, 255)).save(buf, format="PNG")
+
+ data_uri = image_bytes_to_model_data_uri(
+ buf.getvalue(),
+ "large-la.png",
+ config=ImageConfig(
+ preview_max_dimension=64,
+ max_file_size_mb=100.0,
+ large_image_threshold_dimension=100,
+ ),
+ )
+
+ assert data_uri.startswith("data:image/jpeg;base64,")
+ _, encoded = data_uri.split(";base64,", 1)
+ with Image.open(io.BytesIO(base64.b64decode(encoded))) as img:
+ assert img.mode == "RGB"
+ assert max(img.size) <= 64
diff --git a/tests/unit/test_vectorize_file_strategy.py b/tests/unit/test_vectorize_file_strategy.py
index 0e8c5716e..c5ff22d0a 100644
--- a/tests/unit/test_vectorize_file_strategy.py
+++ b/tests/unit/test_vectorize_file_strategy.py
@@ -1,6 +1,9 @@
+import base64
+import io
import types
import pytest
+from PIL import Image
from openviking.core.context import ResourceContentType
from openviking.parse.parsers.media.utils import (
@@ -9,6 +12,7 @@ from openviking.parse.parsers.media.utils import (
)
from openviking.utils import embedding_utils
from openviking.utils.ingest_options import IngestOptions
+from openviking_cli.utils.config.parser_config import ImageConfig
class DummyQueue:
@@ -88,6 +92,18 @@ class DummyReq:
self.account_id = "default"
+def _jpeg_bytes(width: int, height: int) -> bytes:
+ buf = io.BytesIO()
+ Image.new("RGB", (width, height), "white").save(buf, format="JPEG")
+ return buf.getvalue()
+
+
+def _data_uri_image_size(data_uri: str) -> tuple[int, int]:
+ _, encoded = data_uri.split(";base64,", 1)
+ with Image.open(io.BytesIO(base64.b64decode(encoded))) as img:
+ return img.size
+
+
def test_get_resource_content_type_recognizes_media_extensions():
expected = {
"recording.ogg": ResourceContentType.AUDIO,
@@ -172,6 +188,42 @@ async def test_vectorize_file_uses_summary_first(monkeypatch):
assert "content" not in queue.items[0].context_data
+async def test_vectorize_image_downsamples_large_embedding_input(monkeypatch):
+ queue = DummyQueue()
+ original = _jpeg_bytes(80, 220)
+ fs = DummyFS(original)
+ monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue))
+ monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: fs)
+ monkeypatch.setattr(
+ embedding_utils,
+ "get_openviking_config",
+ lambda: types.SimpleNamespace(
+ embedding=types.SimpleNamespace(text_source="content_only", max_input_tokens=1000),
+ image=ImageConfig(
+ preview_max_dimension=64,
+ max_file_size_mb=100.0,
+ large_image_threshold_dimension=100,
+ ),
+ ),
+ )
+
+ await embedding_utils.vectorize_file(
+ file_path="viking://resources/docs/large.jpg",
+ summary_dict={"name": "large.jpg", "summary": "large image"},
+ parent_uri="viking://resources/docs",
+ ctx=DummyReq(),
+ )
+
+ assert len(queue.items) == 1
+ message = queue.items[0].message
+ image_part = next(part for part in message if part["type"] == "image_url")
+ width, height = _data_uri_image_size(image_part["image_url"]["url"])
+ assert max(width, height) <= 64
+ assert fs.content == original
+ assert fs.read_file_bytes_calls == 1
+
+
@pytest.mark.asyncio
async def test_vectorize_file_registers_request_wait_with_embedding_msg_id(monkeypatch):
queue = DummyQueueWithId()