Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(media): downsample oversized image model inputs (#3965)
MaojiaSheng5aed7f72b4b1
diff --git a/openviking/parse/parsers/media/large_image_processor.py b/openviking/parse/parsers/media/large_image_processor.pyindex 6b2bde001..9af149a12 100644--- a/openviking/parse/parsers/media/large_image_processor.py+++ b/openviking/parse/parsers/media/large_image_processor.py@@ -18,6 +18,12 @@ from typing import List, Optional, Tuplefrom PIL import Image, ImageDraw, ImageFont+from openviking.utils.media_limits import (+ DEFAULT_IMAGE_MAX_TILE_DIMENSION_PX,+ DEFAULT_IMAGE_PREVIEW_MAX_DIMENSION,+ DEFAULT_IMAGE_TILE_OVERLAP_PX,+ is_large_image_by_size,+)from openviking_cli.utils.config.parser_config import ImageConfigfrom openviking_cli.utils.logger import get_logger@@ -28,21 +34,10 @@ logger = get_logger(__name__)# Configuration# =============================================================================-# Thresholds for triggering large image processing (fallback defaults when no config)-MAX_FILE_SIZE_MB = 10.0 # 10 MB-LARGE_IMAGE_THRESHOLD_DIMENSION = 4096 # 4096 pixels--# Target limits for individual tiles-MAX_TILE_DIMENSION_PX = 2048 # 2048 pixels-TILE_OVERLAP_PX = 2 # 2 pixels of overlap on each side-# Quality settings for JPEG compressionPREVIEW_QUALITY_START = 85TILE_QUALITY = 90-# Preview dimension fallback (when no config provides preview_max_dimension)-PREVIEW_MAX_DIMENSION = 2048-@dataclassclass TileInfo:@@ -108,11 +103,12 @@ def needs_large_image_processing(Returns:True if large image processing is needed"""- max_file_size_mb = config.max_file_size_mb if config else MAX_FILE_SIZE_MB- max_dimension_px = config.large_image_threshold_dimension if config else LARGE_IMAGE_THRESHOLD_DIMENSION-- file_size_mb = get_image_size_mb(file_path)- return file_size_mb > max_file_size_mb or width > max_dimension_px or height > max_dimension_px+ return is_large_image_by_size(+ file_size_bytes=file_path.stat().st_size,+ width=width,+ height=height,+ config=config,+ )def create_low_res_preview(@@ -131,7 +127,7 @@ def create_low_res_preview(Returns:Preview image bytes in JPEG format"""- max_dimension_px = config.preview_max_dimension if config else PREVIEW_MAX_DIMENSION+ max_dimension_px = config.preview_max_dimension if config else DEFAULT_IMAGE_PREVIEW_MAX_DIMENSION# Work on a copyimg = img.copy()@@ -171,8 +167,10 @@ def calculate_grid_dimensions(Returns:Tuple of (num_rows, num_cols)"""- max_tile_dimension_px = config.max_tile_dimension_px if config else MAX_TILE_DIMENSION_PX- tile_overlap_px = config.tile_overlap_px if config else TILE_OVERLAP_PX+ max_tile_dimension_px = (+ config.max_tile_dimension_px if config else DEFAULT_IMAGE_MAX_TILE_DIMENSION_PX+ )+ tile_overlap_px = config.tile_overlap_px if config else DEFAULT_IMAGE_TILE_OVERLAP_PXeffective_tile = max_tile_dimension_px - tile_overlap_px * 2if effective_tile <= 0:@@ -204,7 +202,7 @@ def calculate_tile_positions(Returns:List of (x1, y1, x2, y2) tuples"""- tile_overlap_px = config.tile_overlap_px if config else TILE_OVERLAP_PX+ tile_overlap_px = config.tile_overlap_px if config else DEFAULT_IMAGE_TILE_OVERLAP_PX# Calculate base tile size without overlapbase_tile_width = ceil(width / cols)@@ -353,8 +351,9 @@ def create_grid_overlay(Returns:Grid overlay image bytes"""- tile_overlap_px = config.tile_overlap_px if config else TILE_OVERLAP_PX- preview_max_dimension = config.preview_max_dimension if config else PREVIEW_MAX_DIMENSION+ preview_max_dimension = (+ config.preview_max_dimension if config else DEFAULT_IMAGE_PREVIEW_MAX_DIMENSION+ )# Resize to a reasonable size for overlay drawingow, oh = img.size@@ -442,9 +441,6 @@ def process_large_image(Returns:LargeImageResult with processing results"""- # Use config values or defaults- max_file_size_mb = config.max_file_size_mb if config else MAX_FILE_SIZE_MB-# Load image if not providedown_img = Falseif img is None:diff --git a/openviking/parse/parsers/media/utils.py b/openviking/parse/parsers/media/utils.pyindex daeb471eb..7ce8e5306 100644--- a/openviking/parse/parsers/media/utils.py+++ b/openviking/parse/parsers/media/utils.py@@ -12,6 +12,7 @@ from openviking.core.path_variables import CalendarVariableProviderfrom openviking.parse.parsers.constants import TYPESCRIPT_MPEG_TS_EXTENSIONfrom openviking.prompts import render_promptfrom openviking.storage.viking_fs import get_viking_fs+from openviking.utils.image_search import prepare_image_bytes_for_modelfrom openviking.utils.media_limits import MAX_MEDIA_FILE_BYTESfrom openviking_cli.utils.config import get_openviking_configfrom openviking_cli.utils.logger import get_logger@@ -264,7 +265,8 @@ async def generate_image_summary(Dictionary with "name" and "summary" keys"""viking_fs = get_viking_fs()- vlm = get_openviking_config().vlm+ config = get_openviking_config()+ vlm = config.vlmfile_name = original_filenametry:@@ -295,7 +297,12 @@ async def generate_image_summary(async with llm_sem or asyncio.Semaphore(1):response = await vlm.get_vision_completion_async(prompt=prompt,- images=[image_bytes],+ images=[+ prepare_image_bytes_for_model(+ image_bytes,+ config=getattr(config, "image", None),+ )+ ],)logger.info(diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.pyindex a34b463d3..24b193030 100644--- a/openviking/utils/embedding_utils.py+++ b/openviking/utils/embedding_utils.py@@ -31,7 +31,7 @@ from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConvefrom openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fsfrom openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.utils.embedding_input import truncate_embedding_input-from openviking.utils.image_search import image_bytes_to_data_uri+from openviking.utils.image_search import image_bytes_to_model_data_urifrom openviking.utils.ingest_options import IngestOptionsfrom openviking.utils.time_utils import parse_iso_datetimefrom openviking_cli.utils import VikingURI, get_logger@@ -272,11 +272,14 @@ async def _build_image_data_uri() -> Optional[str]:"""Read an image file and encode it as a base64 ``data:`` URI.+ Oversized images are downsampled only for the embedding request. The+ original resource bytes in VikingFS are left unchanged.Returns None if the image cannot be read."""try:content = await viking_fs.read_file_bytes(file_path, ctx=ctx)- return image_bytes_to_data_uri(content, file_name)+ image_config = getattr(get_openviking_config(), "image", None)+ return image_bytes_to_model_data_uri(content, file_name, config=image_config)except Exception as e:logger.warning(f"Failed to read image for multimodal vectorization {file_path}: {e}")return None@@ -571,7 +574,9 @@ async def vectorize_file(if summary:context.set_vectorize(Vectorize(text=summary))else:- logger.warning(f"No summary available for {file_path}, skipping vectorization")+ logger.warning(+ f"No summary available for {file_path}, skipping vectorization"+ )return Falseelse:embedding_text = truncate_embedding_input(diff --git a/openviking/utils/image_search.py b/openviking/utils/image_search.pyindex e44eb279c..d5a8b7d1f 100644--- a/openviking/utils/image_search.py+++ b/openviking/utils/image_search.py@@ -5,11 +5,17 @@from __future__ import annotationsimport base64+import ioimport mimetypesimport osfrom pathlib import Pathfrom typing import Any, Dict, List, Optional+from PIL import Image, UnidentifiedImageError++from openviking.utils.media_limits import is_large_image_by_size+from openviking_cli.utils.config.parser_config import ImageConfig+def image_mime_type(file_name: str = "") -> str:mime_type, _ = mimetypes.guess_type(file_name or "")@@ -23,6 +29,57 @@ def image_bytes_to_data_uri(data: bytes | bytearray | memoryview, file_name: strreturn f"data:{image_mime_type(file_name)};base64,{encoded}"+def _prepare_image_bytes_for_model(+ data: bytes | bytearray | memoryview,+ config: ImageConfig | None = None,+) -> tuple[bytes, bool]:+ """Return model-ready image bytes and whether they differ from the input."""+ content = bytes(data)+ try:+ with Image.open(io.BytesIO(content)) as img:+ width, height = img.size+ image_config = config or ImageConfig()+ if not is_large_image_by_size(+ file_size_bytes=len(content),+ width=width,+ height=height,+ config=image_config,+ ):+ return content, False++ preview = img.convert("RGB")++ max_dimension = image_config.preview_max_dimension+ if width > max_dimension or height > max_dimension:+ ratio = min(max_dimension / width, max_dimension / height)+ new_size = (max(1, int(width * ratio)), max(1, int(height * ratio)))+ preview = preview.resize(new_size, Image.Resampling.LANCZOS)++ buf = io.BytesIO()+ preview.save(buf, format="JPEG", quality=85, optimize=True)+ return buf.getvalue(), True+ except (UnidentifiedImageError, OSError, ValueError):+ return content, False+++def prepare_image_bytes_for_model(+ data: bytes | bytearray | memoryview,+ config: ImageConfig | None = None,+) -> bytes:+ """Downsample oversized image bytes for model requests without changing storage."""+ model_bytes, _ = _prepare_image_bytes_for_model(data, config=config)+ return model_bytes+++def image_bytes_to_model_data_uri(+ data: bytes | bytearray | memoryview,+ file_name: str = "",+ config: ImageConfig | None = None,+) -> str:+ model_bytes, changed = _prepare_image_bytes_for_model(data, config=config)+ return image_bytes_to_data_uri(model_bytes, "model_input.jpg" if changed else file_name)++def is_data_image_uri(value: str) -> bool:return value.startswith("data:image/") and ";base64," in valuediff --git a/openviking/utils/media_limits.py b/openviking/utils/media_limits.pyindex d96038730..c64fbba3d 100644--- a/openviking/utils/media_limits.py+++ b/openviking/utils/media_limits.py@@ -1,5 +1,37 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0-"""Shared resource limits for audio and video processing."""+"""Shared media limits and limit checks."""++from typing import ProtocolMAX_MEDIA_FILE_BYTES = 512 * 1024 * 1024++DEFAULT_LARGE_IMAGE_MAX_FILE_SIZE_MB = 10.0+DEFAULT_LARGE_IMAGE_THRESHOLD_DIMENSION = 4096+DEFAULT_IMAGE_PREVIEW_MAX_DIMENSION = 2048+DEFAULT_IMAGE_MAX_TILE_DIMENSION_PX = 2048+DEFAULT_IMAGE_TILE_OVERLAP_PX = 2+++class ImageLimitConfig(Protocol):+ max_file_size_mb: float+ large_image_threshold_dimension: int+++def is_large_image_by_size(+ *,+ file_size_bytes: int,+ width: int,+ height: int,+ config: ImageLimitConfig | None = None,+) -> bool:+ """Return whether image bytes exceed the configured large-image limits."""+ max_file_size_mb = config.max_file_size_mb if config else DEFAULT_LARGE_IMAGE_MAX_FILE_SIZE_MB+ max_dimension_px = (+ config.large_image_threshold_dimension+ if config+ else DEFAULT_LARGE_IMAGE_THRESHOLD_DIMENSION+ )++ file_size_mb = file_size_bytes / (1024 * 1024)+ return file_size_mb > max_file_size_mb or width > max_dimension_px or height > max_dimension_pxdiff --git a/tests/parse/test_media_understanding_summary.py b/tests/parse/test_media_understanding_summary.pyindex 1ebe34cec..7d5fdbce6 100644--- a/tests/parse/test_media_understanding_summary.py+++ b/tests/parse/test_media_understanding_summary.py@@ -1,11 +1,14 @@import asyncio+import iofrom pathlib import Pathfrom types import SimpleNamespacefrom unittest.mock import AsyncMockimport pytest+from PIL import Imagefrom openviking.parse.parsers.media import utils as media_utils+from openviking_cli.utils.config.parser_config import ImageConfigfrom openviking_cli.utils.config.vlm_config import VLMConfig@@ -24,6 +27,9 @@ class _FS:return b""return self.content[offset : offset + size]+ async def read_file_bytes(self, _uri, ctx=None):+ return self.content+class _BlockingReadFS:def __init__(self):@@ -102,6 +108,15 @@ class _CapturingPathClient(_MediaVLM):return "# Clip\n\nUseful summary."+class _ImageVLM:+ def __init__(self):+ self.images = []++ async def get_vision_completion_async(self, *, prompt, images):+ self.images = images+ return "image summary"++def _config(model_config, *, max_chars=4000):if hasattr(model_config, "get_client_instance"):vlm = model_config.get_client_instance()@@ -118,6 +133,17 @@ def _config(model_config, *, max_chars=4000):)+def _jpeg_bytes(width: int, height: int) -> bytes:+ buf = io.BytesIO()+ Image.new("RGB", (width, height), "white").save(buf, format="JPEG")+ return buf.getvalue()+++def _image_size(data: bytes) -> tuple[int, int]:+ with Image.open(io.BytesIO(data)) as img:+ return img.size++def _lazy_client(*, return_value=None, side_effect=None):async def invoke(prepare_media=None, **_kwargs):if prepare_media is not None:@@ -176,6 +202,32 @@ async def test_media_concurrency_bounds_staging_and_inference(monkeypatch):assert fs.peak_reads == 2+async def test_image_summary_downsamples_large_model_input(monkeypatch):+ original = _jpeg_bytes(80, 220)+ fs = _FS(original)+ vlm = _ImageVLM()+ config = SimpleNamespace(+ vlm=vlm,+ image=ImageConfig(+ preview_max_dimension=64,+ max_file_size_mb=100.0,+ large_image_threshold_dimension=100,+ ),+ )+ monkeypatch.setattr(media_utils, "get_viking_fs", lambda: fs)+ monkeypatch.setattr(media_utils, "get_openviking_config", lambda: config)++ result = await media_utils.generate_image_summary(+ "viking://resources/docs/large.jpg",+ "large.jpg",+ )++ assert result == {"name": "large.jpg", "summary": "image summary"}+ assert len(vlm.images) == 1+ assert max(_image_size(vlm.images[0])) <= 64+ assert fs.content == original++async def test_unknown_size_media_stops_at_hard_staging_limit(monkeypatch,):diff --git a/tests/unit/test_image_search.py b/tests/unit/test_image_search.pyindex 070cdcad1..ddcf4d7d5 100644--- a/tests/unit/test_image_search.py+++ b/tests/unit/test_image_search.py@@ -1,9 +1,16 @@+import base64+import io++from PIL import Image+from openviking.models.embedder.base import DenseEmbedderBase, EmbedResultfrom openviking.utils.image_search import (build_multimodal_embedding_input,image_bytes_to_data_uri,+ image_bytes_to_model_data_uri,normalize_client_image_input,)+from openviking_cli.utils.config.parser_config import ImageConfigclass TextOnlyEmbedder(DenseEmbedderBase):@@ -36,3 +43,24 @@ def test_image_helpers_accept_base64_bytes_and_image_uris(tmp_path):assert image_bytes_to_data_uri(b"png").startswith("data:image/png;base64,")assert normalize_client_image_input(image_path).startswith("data:image/jpeg;base64,")+++def test_model_image_input_downsamples_jpeg_incompatible_modes():+ buf = io.BytesIO()+ Image.new("LA", (80, 220), color=(255, 255)).save(buf, format="PNG")++ data_uri = image_bytes_to_model_data_uri(+ buf.getvalue(),+ "large-la.png",+ config=ImageConfig(+ preview_max_dimension=64,+ max_file_size_mb=100.0,+ large_image_threshold_dimension=100,+ ),+ )++ assert data_uri.startswith("data:image/jpeg;base64,")+ _, encoded = data_uri.split(";base64,", 1)+ with Image.open(io.BytesIO(base64.b64decode(encoded))) as img:+ assert img.mode == "RGB"+ assert max(img.size) <= 64diff --git a/tests/unit/test_vectorize_file_strategy.py b/tests/unit/test_vectorize_file_strategy.pyindex 0e8c5716e..c5ff22d0a 100644--- a/tests/unit/test_vectorize_file_strategy.py+++ b/tests/unit/test_vectorize_file_strategy.py@@ -1,6 +1,9 @@+import base64+import ioimport typesimport pytest+from PIL import Imagefrom openviking.core.context import ResourceContentTypefrom openviking.parse.parsers.media.utils import (@@ -9,6 +12,7 @@ from openviking.parse.parsers.media.utils import ()from openviking.utils import embedding_utilsfrom openviking.utils.ingest_options import IngestOptions+from openviking_cli.utils.config.parser_config import ImageConfigclass DummyQueue:@@ -88,6 +92,18 @@ class DummyReq:self.account_id = "default"+def _jpeg_bytes(width: int, height: int) -> bytes:+ buf = io.BytesIO()+ Image.new("RGB", (width, height), "white").save(buf, format="JPEG")+ return buf.getvalue()+++def _data_uri_image_size(data_uri: str) -> tuple[int, int]:+ _, encoded = data_uri.split(";base64,", 1)+ with Image.open(io.BytesIO(base64.b64decode(encoded))) as img:+ return img.size++def test_get_resource_content_type_recognizes_media_extensions():expected = {"recording.ogg": ResourceContentType.AUDIO,@@ -172,6 +188,42 @@ async def test_vectorize_file_uses_summary_first(monkeypatch):assert "content" not in queue.items[0].context_data+async def test_vectorize_image_downsamples_large_embedding_input(monkeypatch):+ queue = DummyQueue()+ original = _jpeg_bytes(80, 220)+ fs = DummyFS(original)+ monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue))+ monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: fs)+ monkeypatch.setattr(+ embedding_utils,+ "get_openviking_config",+ lambda: types.SimpleNamespace(+ embedding=types.SimpleNamespace(text_source="content_only", max_input_tokens=1000),+ image=ImageConfig(+ preview_max_dimension=64,+ max_file_size_mb=100.0,+ large_image_threshold_dimension=100,+ ),+ ),+ )++ await embedding_utils.vectorize_file(+ file_path="viking://resources/docs/large.jpg",+ summary_dict={"name": "large.jpg", "summary": "large image"},+ parent_uri="viking://resources/docs",+ ctx=DummyReq(),+ )++ assert len(queue.items) == 1+ message = queue.items[0].message+ image_part = next(part for part in message if part["type"] == "image_url")+ width, height = _data_uri_image_size(image_part["image_url"]["url"])+ assert max(width, height) <= 64+ assert fs.content == original+ assert fs.read_file_bytes_calls == 1++@pytest.mark.asyncioasync def test_vectorize_file_registers_request_wait_with_embedding_msg_id(monkeypatch):queue = DummyQueueWithId()