Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(vlm): set Ollama num_ctx and disable thinking for local memory extraction (#2992)

t0sakid9a9b3d2e178
diff --git a/openviking/models/vlm/backends/litellm_vlm.py b/openviking/models/vlm/backends/litellm_vlm.py
index 83ca352e3..ade88bf2b 100644
--- a/openviking/models/vlm/backends/litellm_vlm.py
+++ b/openviking/models/vlm/backends/litellm_vlm.py
@@ -98,6 +98,17 @@ PROVIDER_CONFIGS: Dict[str, Dict[str, Any]] = {
}
+# Ollama defaults to a 4096-token context window and silently truncates any
+# longer prompt to fit. OV prompts (memory extraction is ~5k+ tokens) overflow
+# it, so the model never sees the real input and returns empty/garbage with no
+# error. Default to a larger window for Ollama models; callers can override via
+# ``extra_request_body["num_ctx"]``.
+OLLAMA_DEFAULT_NUM_CTX = 16384
+
+# LiteLLM routes that address a local Ollama server.
+OLLAMA_LITELLM_PREFIXES: tuple[str, ...] = ("ollama/", "ollama_chat/")
+
+
# Prefixes that are already complete LiteLLM routes. Keep them authoritative
# so keyword-based auto-detection does not rewrite cross-provider model names.
EXPLICIT_LITELLM_PREFIXES: tuple[str, ...] = (
@@ -288,6 +299,16 @@ class LiteLLMVLMProvider(VLMBase):
if self.extra_request_body:
kwargs["extra_body"] = dict(self.extra_request_body)
+ # Ollama-specific request options. Without an explicit num_ctx the server
+ # truncates long prompts to its 4096-token default; thinking models left
+ # in thinking mode emit only reasoning and stall on CPU. Set safe
+ # defaults, but let extra_request_body override either.
+ if _has_litellm_prefix(model, OLLAMA_LITELLM_PREFIXES):
+ extra = kwargs.get("extra_body", {})
+ extra.setdefault("num_ctx", OLLAMA_DEFAULT_NUM_CTX)
+ extra.setdefault("think", self._effective_thinking(thinking))
+ kwargs["extra_body"] = extra
+
# Only send enable_thinking to DashScope-compatible providers
provider = self._detected_provider or detect_provider_by_model(model)
if provider == "dashscope":
diff --git a/openviking_cli/setup_wizard.py b/openviking_cli/setup_wizard.py
index b7a905632..197f91a8b 100644
--- a/openviking_cli/setup_wizard.py
+++ b/openviking_cli/setup_wizard.py
@@ -655,7 +655,6 @@ EMBEDDING_PRESETS: list[EmbeddingPreset] = [
]
VLM_PRESETS: list[VLMPreset] = [
- VLMPreset("Qwen 3.5 2B", "qwen3.5:2b", "ollama/qwen3.5:2b", "~2.7 GB", 4),
VLMPreset("Qwen 3.5 4B", "qwen3.5:4b", "ollama/qwen3.5:4b", "~3.4 GB", 8),
VLMPreset("Qwen 3.5 9B", "qwen3.5:9b", "ollama/qwen3.5:9b", "~6.6 GB", 16),
VLMPreset("Qwen 3.6 27B", "qwen3.6:27b", "ollama/qwen3.6:27b", "~17 GB, 256K ctx", 32),
@@ -689,16 +688,18 @@ QUERY_PLANNER_PRESETS: list[QueryPlannerPreset] = [
# carries the parameter count, not the artifact size).
_QUERY_PLANNER_DOWNLOAD_GB = 0.9
-# Recommended defaults indexed by RAM tier
+# Recommended defaults indexed by RAM tier. qwen3.5:4b is the smallest VLM we
+# recommend — smaller models fail OV's memory extraction (they copy the prompt's
+# few-shot examples into fabricated memories), so there is no sub-4B tier.
_RAM_TIERS: list[tuple[int, int, int]] = [
# (max_ram_gb, embedding_preset_index, vlm_preset_index)
- (8, 0, 0), # ≤8 GB: qwen3-embedding:0.6b + qwen3.5:2b
- (16, 0, 1), # 8-16 GB: qwen3-embedding:0.6b + qwen3.5:4b
- (32, 2, 2), # 16-32 GB: qwen3-embedding:8b + qwen3.5:9b
- (64, 2, 7), # 32-64 GB: qwen3-embedding:8b + gemma4:e4b
+ (8, 0, 0), # ≤8 GB: qwen3-embedding:0.6b + qwen3.5:4b
+ (16, 0, 0), # 8-16 GB: qwen3-embedding:0.6b + qwen3.5:4b
+ (32, 2, 1), # 16-32 GB: qwen3-embedding:8b + qwen3.5:9b
+ (64, 2, 6), # 32-64 GB: qwen3-embedding:8b + gemma4:e4b
]
_RAM_DEFAULT_EMBED = 2 # ≥64 GB: qwen3-embedding:8b
-_RAM_DEFAULT_VLM = 3 # ≥64 GB: qwen3.6:27b
+_RAM_DEFAULT_VLM = 2 # ≥64 GB: qwen3.6:27b
def _get_recommended_indices(ram_gb: int) -> tuple[int, int]:
@@ -864,7 +865,13 @@ def _ollama_dense_config(embedding: EmbeddingPreset) -> dict[str, Any]:
def _ollama_vlm_config(vlm: VLMPreset) -> dict[str, Any]:
- """Build the VLM config block for an Ollama-served model."""
+ """Build the VLM config block for an Ollama-served model.
+
+ ``extra_request_body`` raises Ollama's context window past its 4096-token
+ default (OV's memory-extraction prompt alone is ~5k tokens, so the default
+ silently truncates the conversation) and disables thinking, which otherwise
+ makes thinking models emit only reasoning and stall.
+ """
return {
"provider": "litellm",
"model": vlm.litellm_model,
@@ -872,6 +879,7 @@ def _ollama_vlm_config(vlm: VLMPreset) -> dict[str, Any]:
"api_base": "http://localhost:11434",
"temperature": 0.0,
"max_retries": 2,
+ "extra_request_body": {"num_ctx": 16384, "think": False},
}
diff --git a/tests/cli/test_setup_wizard.py b/tests/cli/test_setup_wizard.py
index 0b9260fd6..8ac0cf07f 100644
--- a/tests/cli/test_setup_wizard.py
+++ b/tests/cli/test_setup_wizard.py
@@ -133,7 +133,7 @@ class TestModelAvailability:
class TestConfigBuilding:
def test_ollama_config_structure(self):
embedding = EMBEDDING_PRESETS[0] # qwen3-embedding:0.6b
- vlm = VLM_PRESETS[0] # qwen3.5:2b
+ vlm = VLM_PRESETS[0] # qwen3.5:4b
config = _build_ollama_config(embedding, vlm, "/tmp/ov_test")
@@ -147,9 +147,12 @@ class TestConfigBuilding:
vlm_cfg = config["vlm"]
assert vlm_cfg["provider"] == "litellm"
- assert vlm_cfg["model"] == "ollama/qwen3.5:2b"
+ assert vlm_cfg["model"] == "ollama/qwen3.5:4b"
assert vlm_cfg["api_key"] == "no-key"
assert vlm_cfg["api_base"] == "http://localhost:11434"
+ # Ollama needs a larger context window (default 4096 truncates OV's
+ # ~5k-token extraction prompt) and thinking disabled.
+ assert vlm_cfg["extra_request_body"] == {"num_ctx": 16384, "think": False}
def test_cloud_config_structure(self):
provider = CLOUD_PROVIDERS[2] # OpenAI
@@ -611,7 +614,7 @@ class TestRAMRecommendations:
def test_low_ram(self):
emb_idx, vlm_idx = _get_recommended_indices(4)
assert EMBEDDING_PRESETS[emb_idx].model == "qwen3-embedding:0.6b"
- assert VLM_PRESETS[vlm_idx].ollama_model == "qwen3.5:2b"
+ assert VLM_PRESETS[vlm_idx].ollama_model == "qwen3.5:4b"
def test_medium_ram(self):
emb_idx, vlm_idx = _get_recommended_indices(16)
@@ -769,7 +772,7 @@ class TestLocalConfigBuilding:
workspace="/tmp/ov_test",
vlm_config={
"provider": "litellm",
- "model": "ollama/qwen3.5:2b",
+ "model": "ollama/qwen3.5:4b",
"api_key": "no-key",
"api_base": "http://localhost:11434",
},
@@ -777,7 +780,7 @@ class TestLocalConfigBuilding:
assert config["embedding"]["dense"]["provider"] == "local"
assert config["vlm"]["provider"] == "litellm"
- assert config["vlm"]["model"] == "ollama/qwen3.5:2b"
+ assert config["vlm"]["model"] == "ollama/qwen3.5:4b"
def test_local_config_with_cloud_vlm(self):
config = _build_local_config(
diff --git a/tests/unit/test_extra_headers_vlm.py b/tests/unit/test_extra_headers_vlm.py
index c7c12eaf0..9bb0a8ca0 100644
--- a/tests/unit/test_extra_headers_vlm.py
+++ b/tests/unit/test_extra_headers_vlm.py
@@ -516,7 +516,51 @@ class TestVLMExtraRequestBody:
kwargs = vlm._build_text_kwargs(prompt="hello")
- assert kwargs["extra_body"] == {"think": False}
+ # Ollama models also get a default num_ctx; the explicit think is kept.
+ assert kwargs["extra_body"] == {"think": False, "num_ctx": 16384}
+
+ def test_ollama_defaults_num_ctx_and_think(self):
+ """Ollama models get a larger context window and thinking disabled by default."""
+ vlm = LiteLLMVLMProvider(
+ {
+ "model": "ollama/qwen3.5:4b",
+ "provider": "litellm",
+ "api_base": "http://127.0.0.1:11434",
+ }
+ )
+
+ kwargs = vlm._build_text_kwargs(prompt="hello")
+
+ assert kwargs["extra_body"] == {"num_ctx": 16384, "think": False}
+
+ def test_ollama_extra_request_body_overrides_num_ctx(self):
+ """An explicit num_ctx in extra_request_body is not overridden by the default."""
+ vlm = LiteLLMVLMProvider(
+ {
+ "model": "ollama/qwen3.5:4b",
+ "provider": "litellm",
+ "api_base": "http://127.0.0.1:11434",
+ "extra_request_body": {"num_ctx": 32768},
+ }
+ )
+
+ kwargs = vlm._build_text_kwargs(prompt="hello")
+
+ assert kwargs["extra_body"] == {"num_ctx": 32768, "think": False}
+
+ def test_non_ollama_model_gets_no_num_ctx(self):
+ """num_ctx is Ollama-specific and must not leak into other providers."""
+ vlm = LiteLLMVLMProvider(
+ {
+ "model": "gpt-4o-mini",
+ "provider": "litellm",
+ "api_key": "sk-test",
+ }
+ )
+
+ kwargs = vlm._build_text_kwargs(prompt="hello")
+
+ assert "extra_body" not in kwargs or "num_ctx" not in kwargs.get("extra_body", {})
def test_litellm_dashscope_merges_thinking_with_extra_request_body(self):
vlm = LiteLLMVLMProvider(