Personal AI, On Personal Devices
Python
fix(memory): surface clear error when openjarvis_rust missing instead of silent no-op (#527)
Jon Saad-Falconbc9fa6b3c895
diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.tsindex aa2a74b1..19a8f67d 100644--- a/frontend/src/lib/api.ts+++ b/frontend/src/lib/api.ts@@ -948,12 +948,31 @@ export interface MemoryStats {export interface MemoryConfig {backend: string;+ // Set by the server when the native `openjarvis_rust` extension is missing,+ // so the UI can show the real cause instead of a healthy-looking config.+ available?: boolean;+ detail?: string | null;context_from_memory: boolean;context_top_k: number;context_min_score: number;context_max_tokens: number;}+/**+ * Extract the server's `detail` message from a failed JSON response so the UI+ * surfaces the real cause (e.g. "openjarvis_rust extension is not installed")+ * instead of a blanket fallback string (#502).+ */+async function memoryErrorDetail(res: Response, fallback: string): Promise<string> {+ try {+ const data = await res.json();+ if (data && typeof data.detail === 'string' && data.detail) return data.detail;+ } catch {+ // Non-JSON body — fall through to the generic message below.+ }+ return fallback;+}+export async function getMemoryStats(): Promise<MemoryStats> {const res = await apiFetch(`/v1/memory/stats`);if (!res.ok) throw new Error('Failed to fetch memory stats');@@ -977,16 +996,16 @@ export async function storeMemory(content: string, metadata?: Record<string, unkheaders: { 'Content-Type': 'application/json' },body: JSON.stringify({ content, metadata }),});- if (!res.ok) throw new Error('Failed to store memory');+ if (!res.ok) throw new Error(await memoryErrorDetail(res, 'Failed to store memory'));}-export async function indexMemoryPath(path: string): Promise<{ chunks_indexed: number }> {+export async function indexMemoryPath(path: string): Promise<{ chunks_indexed: number; note?: string }> {const res = await apiFetch(`/v1/memory/index`, {method: 'POST',headers: { 'Content-Type': 'application/json' },body: JSON.stringify({ path }),});- if (!res.ok) throw new Error('Failed to index path');+ if (!res.ok) throw new Error(await memoryErrorDetail(res, 'Failed to index path'));return res.json();}diff --git a/scripts/install/build-extension.sh b/scripts/install/build-extension.shindex bbbc05e4..93f04b54 100755--- a/scripts/install/build-extension.sh+++ b/scripts/install/build-extension.sh@@ -28,6 +28,22 @@ ficd "$SRC_DIR"if uv run maturin develop -m "$MANIFEST" >>"$LOG" 2>&1; then+ # Verify the extension actually imports from THIS venv before declaring+ # success. `maturin develop` can report success while installing the .so+ # into a different venv than the one that runs the server, which leaves+ # memory silently broken at runtime (#502). Only the import check below+ # proves the serving venv can load it.+ if ! uv run python -c "import openjarvis_rust" >>"$LOG" 2>&1; then+ rc=$?+ {+ echo "build-extension.sh: maturin succeeded but 'import openjarvis_rust'"+ echo "failed in the serving venv ($SRC_DIR/.venv) — the extension was"+ echo "not installed where the server runs. (exit=$rc)"+ tail -n 50 "$LOG" 2>/dev/null || true+ } > "$FAILED"+ rm -f "$BUILT"+ exit "$rc"+ fitmp="$BUILT.tmp"date -u +"%Y-%m-%dT%H:%M:%SZ" > "$tmp"mv "$tmp" "$BUILT"diff --git a/src/openjarvis/server/api_routes.py b/src/openjarvis/server/api_routes.pyindex 3b2ff650..0d29a005 100644--- a/src/openjarvis/server/api_routes.py+++ b/src/openjarvis/server/api_routes.py@@ -153,14 +153,30 @@ memory_router = APIRouter(prefix="/v1/memory", tags=["memory"])def _get_memory_backend(request: Request):- """Return the app-level memory backend, falling back to a fresh SQLiteMemory."""+ """Return the app-level memory backend, falling back to a fresh SQLiteMemory.++ Raises ``HTTPException(503)`` with an actionable message when the backend+ cannot be built because the mandatory ``openjarvis_rust`` extension is not+ installed in the serving venv. This is deliberately distinct from a benign+ "memory not configured" case (which returns ``None``): a missing native+ extension must fail loudly, never silently degrade (#502).+ """backend = getattr(request.app.state, "memory_backend", None)if backend is None:+ from openjarvis.tools.storage._stubs import MemoryBackendUnavailable+try:from openjarvis.tools.storage.sqlite import SQLiteMemorybackend = SQLiteMemory()+ except MemoryBackendUnavailable as exc:+ # The native extension is missing — surface a loud, actionable error+ # rather than a misleading "no backend" / silent no-op.+ logger.error("%s", exc)+ raise HTTPException(status_code=503, detail=str(exc)) from excexcept Exception:+ # Memory is genuinely unconfigured for a benign reason — preserve+ # the existing graceful "no backend" behaviour.return Nonereturn backend@@ -170,7 +186,9 @@ async def memory_store(req: MemoryStoreRequest, request: Request):"""Store content in memory."""backend = _get_memory_backend(request)if backend is None:- return {"status": "stored", "note": "no backend available"}+ # Memory is intentionally disabled; report it honestly instead of a+ # 200 that silently discards the write (#502).+ raise HTTPException(status_code=503, detail="Memory is not configured")try:backend.store(req.content, metadata=req.metadata or {})return {"status": "stored"}@@ -216,7 +234,13 @@ async def memory_stats(request: Request):@memory_router.get("/config")async def memory_config(request: Request):- """Return current memory configuration."""+ """Return current memory configuration.++ Reports memory as *unavailable* (rather than falsely claiming+ ``backend_type: sqlite``) when the native ``openjarvis_rust`` extension is+ missing, so the UI can show the real cause instead of a healthy-looking+ config that backs a silent no-op (#502).+ """try:config = getattr(request.app.state, "config", None)if config is None:@@ -224,12 +248,30 @@ async def memory_config(request: Request):config = load_config()backend = getattr(request.app.state, "memory_backend", None)+ available = True+ detail: Optional[str] = None+ if backend is None:+ from openjarvis.tools.storage._stubs import MemoryBackendUnavailable++ try:+ from openjarvis.tools.storage.sqlite import SQLiteMemory++ backend = SQLiteMemory()+ except MemoryBackendUnavailable as exc:+ available = False+ detail = str(exc)+ except Exception:+ # Benign: cannot construct a probe backend here, but the+ # configured default is still what would be used.+ passreturn {"backend_type": (backend.backend_idif backend is not Noneelse config.memory.default_backend),+ "available": available,+ "detail": detail,"context_top_k": config.memory.context_top_k,"context_min_score": config.memory.context_min_score,"context_max_tokens": config.memory.context_max_tokens,@@ -253,7 +295,7 @@ async def memory_index(req: MemoryIndexRequest, request: Request):backend = _get_memory_backend(request)if backend is None:- raise HTTPException(status_code=503, detail="No memory backend available")+ raise HTTPException(status_code=503, detail="Memory is not configured")chunks = ingest_path(target)stored = 0@@ -264,7 +306,16 @@ async def memory_index(req: MemoryIndexRequest, request: Request):backend.store(chunk.content, metadata=metadata)stored += 1- return {"status": "indexed", "chunks_indexed": stored}+ result = {"status": "indexed", "chunks_indexed": stored}+ if stored == 0:+ # "indexed" must never silently mean "stored nothing". Surface why+ # so a folder of short notes doesn't look like a successful no-op+ # (#502 follow-up).+ result["note"] = (+ "no content was indexed — the path contained no readable "+ "documents with indexable text"+ )+ return resultexcept HTTPException:raiseexcept Exception as exc:diff --git a/src/openjarvis/tools/storage/_stubs.py b/src/openjarvis/tools/storage/_stubs.pyindex d87593bc..d9280878 100644--- a/src/openjarvis/tools/storage/_stubs.py+++ b/src/openjarvis/tools/storage/_stubs.py@@ -10,6 +10,32 @@ from abc import ABC, abstractmethodfrom dataclasses import dataclass, fieldfrom typing import Any, Dict, List, Optional+#: Actionable message surfaced whenever a Rust-backed memory backend cannot be+#: constructed because the mandatory ``openjarvis_rust`` extension is missing+#: from the *current* venv. Kept as a single constant so the server routes, the+#: SDK and the regression tests all surface exactly the same wording.+RUST_MISSING_HINT = (+ "Memory backend unavailable: the native `openjarvis_rust` extension is not "+ "installed in this environment. Build it into the venv that runs the server "+ "with `uv run maturin develop -m rust/crates/openjarvis-python/Cargo.toml` "+ "(needs rustc >= 1.88), then restart. Verify with "+ '`python -c "from openjarvis._rust_bridge import RUST_AVAILABLE; '+ 'print(RUST_AVAILABLE)"`.'+)+++class MemoryBackendUnavailable(RuntimeError):+ """Raised when a memory backend cannot be built because the mandatory+ ``openjarvis_rust`` extension is missing from the current environment.++ This is deliberately distinct from "memory is intentionally disabled": a+ missing native extension is an environment/install error that must be+ surfaced loudly and actionably, never swallowed into a silent no-op.+ """++ def __init__(self, message: str = RUST_MISSING_HINT) -> None:+ super().__init__(message)+@dataclass(slots=True)class RetrievalResult:@@ -59,4 +85,9 @@ class MemoryBackend(ABC):"""Remove all stored documents."""-__all__ = ["MemoryBackend", "RetrievalResult"]+__all__ = [+ "RUST_MISSING_HINT",+ "MemoryBackend",+ "MemoryBackendUnavailable",+ "RetrievalResult",+]diff --git a/src/openjarvis/tools/storage/chunking.py b/src/openjarvis/tools/storage/chunking.pyindex 70e7ada5..abdbdaab 100644--- a/src/openjarvis/tools/storage/chunking.py+++ b/src/openjarvis/tools/storage/chunking.py@@ -135,10 +135,17 @@ def chunk_text(current_tokens.extend(para_tokens)current_offset += len(para_tokens)- # Flush remaining tokens+ # Flush remaining tokens.+ #+ # ``min_chunk_size`` exists to discard tiny *trailing* fragments once a+ # document has already produced at least one chunk. It must NOT silently+ # drop an entire short document: indexing a folder of short notes would+ # otherwise report success while storing nothing (#502 follow-up). So if no+ # chunk has been emitted yet, keep the remaining content regardless of the+ # floor.if current_tokens:chunk_content = " ".join(current_tokens)- if _count_tokens(chunk_content) >= cfg.min_chunk_size:+ if not chunks or _count_tokens(chunk_content) >= cfg.min_chunk_size:chunks.append(Chunk(content=chunk_content,diff --git a/src/openjarvis/tools/storage/sqlite.py b/src/openjarvis/tools/storage/sqlite.pyindex e9d77172..707093c3 100644--- a/src/openjarvis/tools/storage/sqlite.py+++ b/src/openjarvis/tools/storage/sqlite.py@@ -9,7 +9,11 @@ from typing import Any, Dict, List, Optionalfrom openjarvis.core.events import EventType, get_event_busfrom openjarvis.core.registry import MemoryRegistry-from openjarvis.tools.storage._stubs import MemoryBackend, RetrievalResult+from openjarvis.tools.storage._stubs import (+ MemoryBackend,+ MemoryBackendUnavailable,+ RetrievalResult,+)def _check_fts5(conn: sqlite3.Connection) -> bool:@@ -40,7 +44,15 @@ class SQLiteMemory(MemoryBackend):from openjarvis._rust_bridge import get_rust_module- _rust = get_rust_module()+ # The Rust backend is mandatory and there is no Python fallback. When+ # the extension is missing from *this* venv, ``get_rust_module`` raises+ # ImportError; translate it into a clear, actionable error so callers+ # never degrade to a misleading "Failed to index path" or a silent+ # no-op (see #502).+ try:+ _rust = get_rust_module()+ except ImportError as exc:+ raise MemoryBackendUnavailable() from excself._rust_impl = _rust.SQLiteMemory(self._db_path)self._conn = None # type: ignore[assignment]diff --git a/tests/memory/test_chunking.py b/tests/memory/test_chunking.pyindex 35fe3a01..7f6b4741 100644--- a/tests/memory/test_chunking.py+++ b/tests/memory/test_chunking.py@@ -66,13 +66,40 @@ def test_custom_config():assert len(chunks) >= 3-def test_min_chunk_size_filters_tiny():+def test_short_only_document_not_dropped():+ """A whole document below min_chunk_size must still produce a chunk.++ Regression for #502 follow-up: previously a folder of short notes indexed+ to ``chunks_indexed: 0`` (HTTP 200), silently storing nothing. ``min_chunk_size``+ should only discard tiny *trailing fragments*, never an entire short doc.+ """cfg = ChunkConfig(chunk_size=100, chunk_overlap=0, min_chunk_size=50)- # 30 words is below min_chunk_size=50+ # 30 words is below min_chunk_size=50, but it's the entire document.words = [f"w{i}" for i in range(30)]text = " ".join(words)chunks = chunk_text(text, config=cfg)- assert len(chunks) == 0+ assert len(chunks) == 1+ assert chunks[0].content == text+++def test_short_real_world_note_not_dropped():+ """The exact repro from the issue: a ~4-word note must not vanish."""+ chunks = chunk_text("hello world\nsome content\n", source="a.txt")+ assert len(chunks) == 1+ assert "hello world" in chunks[0].content+++def test_min_chunk_size_filters_tiny_trailing_fragment():+ """A tiny fragment trailing a real chunk is still dropped by the floor."""+ cfg = ChunkConfig(chunk_size=50, chunk_overlap=0, min_chunk_size=10)+ # Two paragraphs: the first fills a real chunk, the second is a tiny tail.+ para1 = " ".join(f"a{i}" for i in range(50))+ para2 = " ".join(f"b{i}" for i in range(3)) # 3 words < min_chunk_size=10+ text = f"{para1}\n\n{para2}"+ chunks = chunk_text(text, config=cfg)+ # The 3-word trailing fragment is discarded; only the real chunk remains.+ assert len(chunks) == 1+ assert "b0" not in chunks[0].contentdef test_source_propagated():diff --git a/tests/server/test_api_routes.py b/tests/server/test_api_routes.pyindex d846c348..985e93b6 100644--- a/tests/server/test_api_routes.py+++ b/tests/server/test_api_routes.py@@ -49,6 +49,54 @@ class TestMemoryRoutes:assert resp.status_code in (200, 500)+class TestMemoryRustMissing:+ """Regression for #502: when the native ``openjarvis_rust`` extension is+ missing from the serving venv, memory ops must surface a CLEAR, ACTIONABLE+ error — never the misleading "Failed to index path" or a 200 silent no-op.+ """++ @staticmethod+ def _client(monkeypatch):+ # Force the same failure mode as a venv without the compiled extension.+ def _boom():+ raise ImportError("No module named 'openjarvis_rust'")++ import openjarvis._rust_bridge as bridge++ monkeypatch.setattr(bridge, "get_rust_module", _boom)+ return TestClient(_make_app())++ def test_store_is_not_a_silent_noop(self, monkeypatch):+ client = self._client(monkeypatch)+ resp = client.post("/v1/memory/store", json={"content": "hi"})+ # Must NOT return the old 200 {"status":"stored","note":"no backend..."}.+ assert resp.status_code == 503+ detail = resp.json()["detail"]+ assert "openjarvis_rust" in detail+ assert "maturin develop" in detail++ def test_index_surfaces_actionable_detail(self, monkeypatch, tmp_path):+ (tmp_path / "note.txt").write_text("hello world some content here")+ client = self._client(monkeypatch)+ resp = client.post("/v1/memory/index", json={"path": str(tmp_path)})+ assert resp.status_code == 503+ detail = resp.json()["detail"]+ # The frontend reads this `detail`; it must point at the real cause,+ # not blame the indexed path.+ assert "openjarvis_rust" in detail+ assert detail != "Failed to index path"+ assert detail != "No memory backend available"++ def test_config_reports_unavailable(self, monkeypatch):+ client = self._client(monkeypatch)+ resp = client.get("/v1/memory/config")+ assert resp.status_code == 200+ data = resp.json()+ # Must not falsely report a healthy backend when none could be built.+ assert data["available"] is False+ assert "openjarvis_rust" in (data["detail"] or "")++class TestBudgetRoutes:def test_get_budget(self):client = TestClient(_make_app())