Skip to content

Personal AI, On Personal Devices

Python

feat: implement gemma_cpp engine lifecycle and inference methods

Jon Saad-Falcon8c2ee6843ad2
diff --git a/src/openjarvis/engine/gemma_cpp.py b/src/openjarvis/engine/gemma_cpp.py
index a235417b..4e2f8245 100644
--- a/src/openjarvis/engine/gemma_cpp.py
+++ b/src/openjarvis/engine/gemma_cpp.py
@@ -14,6 +14,12 @@ from openjarvis.engine._base import InferenceEngine, estimate_prompt_tokens
logger = logging.getLogger(__name__)
+def _import_pygemma():
+ """Import and return the pygemma.Gemma class. Raises ImportError if unavailable."""
+ from pygemma import Gemma
+ return Gemma
+
+
@EngineRegistry.register("gemma_cpp")
class GemmaCppEngine(InferenceEngine):
"""gemma.cpp backend via pygemma pybind11 bindings (in-process, CPU)."""
@@ -64,6 +70,35 @@ class GemmaCppEngine(InferenceEngine):
parts.append("<start_of_turn>model\n")
return "".join(parts)
+ def _ensure_loaded(self) -> None:
+ """Lazy model loading — called before inference."""
+ if self._gemma is None:
+ if not self._model_path:
+ raise FileNotFoundError(
+ "gemma.cpp model_path not configured. Download weights "
+ "from Kaggle and set GEMMA_CPP_MODEL_PATH or configure "
+ "[engine.gemma_cpp] in ~/.openjarvis/config.toml"
+ )
+ if not self._tokenizer_path:
+ raise FileNotFoundError(
+ "gemma.cpp tokenizer_path not configured. Set "
+ "GEMMA_CPP_TOKENIZER_PATH or configure "
+ "[engine.gemma_cpp] in ~/.openjarvis/config.toml"
+ )
+ Gemma = _import_pygemma()
+ self._gemma = Gemma()
+ self._gemma.load_model(
+ self._tokenizer_path, self._model_path, self._model_type
+ )
+
+ def prepare(self, model: str) -> None:
+ """Load model into memory."""
+ self._ensure_loaded()
+
+ def close(self) -> None:
+ """Unload model and free memory."""
+ self._gemma = None
+
def generate(
self,
messages: Sequence[Message],
@@ -73,7 +108,31 @@ class GemmaCppEngine(InferenceEngine):
max_tokens: int = 1024,
**kwargs: Any,
) -> Dict[str, Any]:
- raise NotImplementedError # implemented in Task 4
+ self._ensure_loaded()
+ if model != self._model_type:
+ logger.warning(
+ "gemma_cpp: requested model %r but loaded model is %r; "
+ "proceeding with loaded model",
+ model, self._model_type,
+ )
+ prompt = self._messages_to_prompt(messages)
+ try:
+ raw = self._gemma.completion(prompt)
+ except Exception as exc:
+ raise RuntimeError(f"gemma.cpp inference failed: {exc}") from exc
+
+ prompt_tokens = estimate_prompt_tokens(messages)
+ completion_tokens = max(1, len(raw) // 4)
+ return {
+ "content": raw,
+ "usage": {
+ "prompt_tokens": prompt_tokens,
+ "completion_tokens": completion_tokens,
+ "total_tokens": prompt_tokens + completion_tokens,
+ },
+ "model": self._model_type,
+ "finish_reason": "stop",
+ }
async def stream(
self,
@@ -84,8 +143,13 @@ class GemmaCppEngine(InferenceEngine):
max_tokens: int = 1024,
**kwargs: Any,
) -> AsyncIterator[str]:
- raise NotImplementedError # implemented in Task 4
- yield "" # pragma: no cover
+ self._ensure_loaded()
+ prompt = self._messages_to_prompt(messages)
+ try:
+ raw = self._gemma.completion(prompt)
+ except Exception as exc:
+ raise RuntimeError(f"gemma.cpp inference failed: {exc}") from exc
+ yield raw
def list_models(self) -> List[str]:
raise NotImplementedError # implemented in Task 5
@@ -93,11 +157,5 @@ class GemmaCppEngine(InferenceEngine):
def health(self) -> bool:
raise NotImplementedError # implemented in Task 5
- def close(self) -> None:
- pass # implemented in Task 4
-
- def prepare(self, model: str) -> None:
- pass # implemented in Task 4
-
__all__ = ["GemmaCppEngine"]
diff --git a/tests/engine/test_gemma_cpp.py b/tests/engine/test_gemma_cpp.py
index 91f8bcfb..82d6b146 100644
--- a/tests/engine/test_gemma_cpp.py
+++ b/tests/engine/test_gemma_cpp.py
@@ -90,3 +90,128 @@ class TestMessagesToPrompt:
"Rule 1\n\nRule 2\n\nGo<end_of_turn>\n"
"<start_of_turn>model\n"
)
+
+
+from unittest.mock import MagicMock, patch
+
+
+class TestGemmaCppLifecycle:
+ def _make_engine(self, **kwargs):
+ from openjarvis.engine.gemma_cpp import GemmaCppEngine
+ defaults = {
+ "model_path": "/fake/model.sbs",
+ "tokenizer_path": "/fake/tokenizer.spm",
+ "model_type": "2b-it",
+ }
+ defaults.update(kwargs)
+ return GemmaCppEngine(**defaults)
+
+ @patch("openjarvis.engine.gemma_cpp._import_pygemma")
+ def test_prepare_loads_model(self, mock_import) -> None:
+ mock_gemma_cls = MagicMock()
+ mock_import.return_value = mock_gemma_cls
+ engine = self._make_engine()
+ engine.prepare("2b-it")
+ mock_gemma_cls.assert_called_once()
+ mock_gemma_cls.return_value.load_model.assert_called_once_with(
+ "/fake/tokenizer.spm", "/fake/model.sbs", "2b-it"
+ )
+
+ @patch("openjarvis.engine.gemma_cpp._import_pygemma")
+ def test_prepare_idempotent(self, mock_import) -> None:
+ mock_gemma_cls = MagicMock()
+ mock_import.return_value = mock_gemma_cls
+ engine = self._make_engine()
+ engine.prepare("2b-it")
+ engine.prepare("2b-it")
+ # Only loaded once
+ assert mock_gemma_cls.call_count == 1
+
+ @patch("openjarvis.engine.gemma_cpp._import_pygemma")
+ def test_close_unloads_model(self, mock_import) -> None:
+ mock_gemma_cls = MagicMock()
+ mock_import.return_value = mock_gemma_cls
+ engine = self._make_engine()
+ engine.prepare("2b-it")
+ assert engine._gemma is not None
+ engine.close()
+ assert engine._gemma is None
+
+ @patch("openjarvis.engine.gemma_cpp._import_pygemma")
+ def test_generate_returns_dict(self, mock_import) -> None:
+ mock_gemma_cls = MagicMock()
+ mock_gemma_instance = MagicMock()
+ mock_gemma_instance.completion.return_value = "The answer is 4."
+ mock_gemma_cls.return_value = mock_gemma_instance
+ mock_import.return_value = mock_gemma_cls
+
+ engine = self._make_engine()
+ result = engine.generate(
+ [Message(role=Role.USER, content="What is 2+2?")],
+ model="2b-it",
+ )
+ assert result["content"] == "The answer is 4."
+ assert "usage" in result
+ assert result["usage"]["prompt_tokens"] > 0
+ assert result["usage"]["completion_tokens"] > 0
+ assert result["usage"]["total_tokens"] > 0
+ assert result["model"] == "2b-it"
+ assert result["finish_reason"] == "stop"
+
+ @patch("openjarvis.engine.gemma_cpp._import_pygemma")
+ def test_generate_warns_on_model_mismatch(self, mock_import) -> None:
+ mock_gemma_cls = MagicMock()
+ mock_gemma_instance = MagicMock()
+ mock_gemma_instance.completion.return_value = "ok"
+ mock_gemma_cls.return_value = mock_gemma_instance
+ mock_import.return_value = mock_gemma_cls
+
+ engine = self._make_engine(model_type="2b-it")
+ from openjarvis.engine import gemma_cpp as gc_mod
+ with patch.object(gc_mod.logger, "warning") as mock_warn:
+ engine.generate(
+ [Message(role=Role.USER, content="Hi")],
+ model="9b-it",
+ )
+ mock_warn.assert_called_once()
+
+ @patch("openjarvis.engine.gemma_cpp._import_pygemma")
+ def test_generate_wraps_runtime_error(self, mock_import) -> None:
+ mock_gemma_cls = MagicMock()
+ mock_gemma_instance = MagicMock()
+ mock_gemma_instance.completion.side_effect = Exception("segfault")
+ mock_gemma_cls.return_value = mock_gemma_instance
+ mock_import.return_value = mock_gemma_cls
+
+ engine = self._make_engine()
+ with pytest.raises(RuntimeError, match="gemma.cpp inference failed"):
+ engine.generate(
+ [Message(role=Role.USER, content="Hi")],
+ model="2b-it",
+ )
+
+
+class TestGemmaCppStream:
+ @patch("openjarvis.engine.gemma_cpp._import_pygemma")
+ @pytest.mark.asyncio
+ async def test_stream_yields_content(self, mock_import) -> None:
+ mock_gemma_cls = MagicMock()
+ mock_gemma_instance = MagicMock()
+ mock_gemma_instance.completion.return_value = "streamed output"
+ mock_gemma_cls.return_value = mock_gemma_instance
+ mock_import.return_value = mock_gemma_cls
+
+ from openjarvis.engine.gemma_cpp import GemmaCppEngine
+ engine = GemmaCppEngine(
+ model_path="/fake/model.sbs",
+ tokenizer_path="/fake/tokenizer.spm",
+ model_type="2b-it",
+ )
+ chunks = []
+ async for chunk in engine.stream(
+ [Message(role=Role.USER, content="Hi")],
+ model="2b-it",
+ ):
+ chunks.append(chunk)
+ assert len(chunks) == 1
+ assert chunks[0] == "streamed output"