Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(vlm): set Ollama num_ctx and disable thinking for local memory extraction (#2992)
t0sakid9a9b3d2e178
diff --git a/openviking/models/vlm/backends/litellm_vlm.py b/openviking/models/vlm/backends/litellm_vlm.pyindex 83ca352e3..ade88bf2b 100644--- a/openviking/models/vlm/backends/litellm_vlm.py+++ b/openviking/models/vlm/backends/litellm_vlm.py@@ -98,6 +98,17 @@ PROVIDER_CONFIGS: Dict[str, Dict[str, Any]] = {}+# Ollama defaults to a 4096-token context window and silently truncates any+# longer prompt to fit. OV prompts (memory extraction is ~5k+ tokens) overflow+# it, so the model never sees the real input and returns empty/garbage with no+# error. Default to a larger window for Ollama models; callers can override via+# ``extra_request_body["num_ctx"]``.+OLLAMA_DEFAULT_NUM_CTX = 16384++# LiteLLM routes that address a local Ollama server.+OLLAMA_LITELLM_PREFIXES: tuple[str, ...] = ("ollama/", "ollama_chat/")++# Prefixes that are already complete LiteLLM routes. Keep them authoritative# so keyword-based auto-detection does not rewrite cross-provider model names.EXPLICIT_LITELLM_PREFIXES: tuple[str, ...] = (@@ -288,6 +299,16 @@ class LiteLLMVLMProvider(VLMBase):if self.extra_request_body:kwargs["extra_body"] = dict(self.extra_request_body)+ # Ollama-specific request options. Without an explicit num_ctx the server+ # truncates long prompts to its 4096-token default; thinking models left+ # in thinking mode emit only reasoning and stall on CPU. Set safe+ # defaults, but let extra_request_body override either.+ if _has_litellm_prefix(model, OLLAMA_LITELLM_PREFIXES):+ extra = kwargs.get("extra_body", {})+ extra.setdefault("num_ctx", OLLAMA_DEFAULT_NUM_CTX)+ extra.setdefault("think", self._effective_thinking(thinking))+ kwargs["extra_body"] = extra+# Only send enable_thinking to DashScope-compatible providersprovider = self._detected_provider or detect_provider_by_model(model)if provider == "dashscope":diff --git a/openviking_cli/setup_wizard.py b/openviking_cli/setup_wizard.pyindex b7a905632..197f91a8b 100644--- a/openviking_cli/setup_wizard.py+++ b/openviking_cli/setup_wizard.py@@ -655,7 +655,6 @@ EMBEDDING_PRESETS: list[EmbeddingPreset] = []VLM_PRESETS: list[VLMPreset] = [- VLMPreset("Qwen 3.5 2B", "qwen3.5:2b", "ollama/qwen3.5:2b", "~2.7 GB", 4),VLMPreset("Qwen 3.5 4B", "qwen3.5:4b", "ollama/qwen3.5:4b", "~3.4 GB", 8),VLMPreset("Qwen 3.5 9B", "qwen3.5:9b", "ollama/qwen3.5:9b", "~6.6 GB", 16),VLMPreset("Qwen 3.6 27B", "qwen3.6:27b", "ollama/qwen3.6:27b", "~17 GB, 256K ctx", 32),@@ -689,16 +688,18 @@ QUERY_PLANNER_PRESETS: list[QueryPlannerPreset] = [# carries the parameter count, not the artifact size)._QUERY_PLANNER_DOWNLOAD_GB = 0.9-# Recommended defaults indexed by RAM tier+# Recommended defaults indexed by RAM tier. qwen3.5:4b is the smallest VLM we+# recommend — smaller models fail OV's memory extraction (they copy the prompt's+# few-shot examples into fabricated memories), so there is no sub-4B tier._RAM_TIERS: list[tuple[int, int, int]] = [# (max_ram_gb, embedding_preset_index, vlm_preset_index)- (8, 0, 0), # ≤8 GB: qwen3-embedding:0.6b + qwen3.5:2b- (16, 0, 1), # 8-16 GB: qwen3-embedding:0.6b + qwen3.5:4b- (32, 2, 2), # 16-32 GB: qwen3-embedding:8b + qwen3.5:9b- (64, 2, 7), # 32-64 GB: qwen3-embedding:8b + gemma4:e4b+ (8, 0, 0), # ≤8 GB: qwen3-embedding:0.6b + qwen3.5:4b+ (16, 0, 0), # 8-16 GB: qwen3-embedding:0.6b + qwen3.5:4b+ (32, 2, 1), # 16-32 GB: qwen3-embedding:8b + qwen3.5:9b+ (64, 2, 6), # 32-64 GB: qwen3-embedding:8b + gemma4:e4b]_RAM_DEFAULT_EMBED = 2 # ≥64 GB: qwen3-embedding:8b-_RAM_DEFAULT_VLM = 3 # ≥64 GB: qwen3.6:27b+_RAM_DEFAULT_VLM = 2 # ≥64 GB: qwen3.6:27bdef _get_recommended_indices(ram_gb: int) -> tuple[int, int]:@@ -864,7 +865,13 @@ def _ollama_dense_config(embedding: EmbeddingPreset) -> dict[str, Any]:def _ollama_vlm_config(vlm: VLMPreset) -> dict[str, Any]:- """Build the VLM config block for an Ollama-served model."""+ """Build the VLM config block for an Ollama-served model.++ ``extra_request_body`` raises Ollama's context window past its 4096-token+ default (OV's memory-extraction prompt alone is ~5k tokens, so the default+ silently truncates the conversation) and disables thinking, which otherwise+ makes thinking models emit only reasoning and stall.+ """return {"provider": "litellm","model": vlm.litellm_model,@@ -872,6 +879,7 @@ def _ollama_vlm_config(vlm: VLMPreset) -> dict[str, Any]:"api_base": "http://localhost:11434","temperature": 0.0,"max_retries": 2,+ "extra_request_body": {"num_ctx": 16384, "think": False},}diff --git a/tests/cli/test_setup_wizard.py b/tests/cli/test_setup_wizard.pyindex 0b9260fd6..8ac0cf07f 100644--- a/tests/cli/test_setup_wizard.py+++ b/tests/cli/test_setup_wizard.py@@ -133,7 +133,7 @@ class TestModelAvailability:class TestConfigBuilding:def test_ollama_config_structure(self):embedding = EMBEDDING_PRESETS[0] # qwen3-embedding:0.6b- vlm = VLM_PRESETS[0] # qwen3.5:2b+ vlm = VLM_PRESETS[0] # qwen3.5:4bconfig = _build_ollama_config(embedding, vlm, "/tmp/ov_test")@@ -147,9 +147,12 @@ class TestConfigBuilding:vlm_cfg = config["vlm"]assert vlm_cfg["provider"] == "litellm"- assert vlm_cfg["model"] == "ollama/qwen3.5:2b"+ assert vlm_cfg["model"] == "ollama/qwen3.5:4b"assert vlm_cfg["api_key"] == "no-key"assert vlm_cfg["api_base"] == "http://localhost:11434"+ # Ollama needs a larger context window (default 4096 truncates OV's+ # ~5k-token extraction prompt) and thinking disabled.+ assert vlm_cfg["extra_request_body"] == {"num_ctx": 16384, "think": False}def test_cloud_config_structure(self):provider = CLOUD_PROVIDERS[2] # OpenAI@@ -611,7 +614,7 @@ class TestRAMRecommendations:def test_low_ram(self):emb_idx, vlm_idx = _get_recommended_indices(4)assert EMBEDDING_PRESETS[emb_idx].model == "qwen3-embedding:0.6b"- assert VLM_PRESETS[vlm_idx].ollama_model == "qwen3.5:2b"+ assert VLM_PRESETS[vlm_idx].ollama_model == "qwen3.5:4b"def test_medium_ram(self):emb_idx, vlm_idx = _get_recommended_indices(16)@@ -769,7 +772,7 @@ class TestLocalConfigBuilding:workspace="/tmp/ov_test",vlm_config={"provider": "litellm",- "model": "ollama/qwen3.5:2b",+ "model": "ollama/qwen3.5:4b","api_key": "no-key","api_base": "http://localhost:11434",},@@ -777,7 +780,7 @@ class TestLocalConfigBuilding:assert config["embedding"]["dense"]["provider"] == "local"assert config["vlm"]["provider"] == "litellm"- assert config["vlm"]["model"] == "ollama/qwen3.5:2b"+ assert config["vlm"]["model"] == "ollama/qwen3.5:4b"def test_local_config_with_cloud_vlm(self):config = _build_local_config(diff --git a/tests/unit/test_extra_headers_vlm.py b/tests/unit/test_extra_headers_vlm.pyindex c7c12eaf0..9bb0a8ca0 100644--- a/tests/unit/test_extra_headers_vlm.py+++ b/tests/unit/test_extra_headers_vlm.py@@ -516,7 +516,51 @@ class TestVLMExtraRequestBody:kwargs = vlm._build_text_kwargs(prompt="hello")- assert kwargs["extra_body"] == {"think": False}+ # Ollama models also get a default num_ctx; the explicit think is kept.+ assert kwargs["extra_body"] == {"think": False, "num_ctx": 16384}++ def test_ollama_defaults_num_ctx_and_think(self):+ """Ollama models get a larger context window and thinking disabled by default."""+ vlm = LiteLLMVLMProvider(+ {+ "model": "ollama/qwen3.5:4b",+ "provider": "litellm",+ "api_base": "http://127.0.0.1:11434",+ }+ )++ kwargs = vlm._build_text_kwargs(prompt="hello")++ assert kwargs["extra_body"] == {"num_ctx": 16384, "think": False}++ def test_ollama_extra_request_body_overrides_num_ctx(self):+ """An explicit num_ctx in extra_request_body is not overridden by the default."""+ vlm = LiteLLMVLMProvider(+ {+ "model": "ollama/qwen3.5:4b",+ "provider": "litellm",+ "api_base": "http://127.0.0.1:11434",+ "extra_request_body": {"num_ctx": 32768},+ }+ )++ kwargs = vlm._build_text_kwargs(prompt="hello")++ assert kwargs["extra_body"] == {"num_ctx": 32768, "think": False}++ def test_non_ollama_model_gets_no_num_ctx(self):+ """num_ctx is Ollama-specific and must not leak into other providers."""+ vlm = LiteLLMVLMProvider(+ {+ "model": "gpt-4o-mini",+ "provider": "litellm",+ "api_key": "sk-test",+ }+ )++ kwargs = vlm._build_text_kwargs(prompt="hello")++ assert "extra_body" not in kwargs or "num_ctx" not in kwargs.get("extra_body", {})def test_litellm_dashscope_merges_thinking_with_extra_request_body(self):vlm = LiteLLMVLMProvider(