Personal AI, On Personal Devices
Python
feat: implement gemma_cpp engine lifecycle and inference methods
Jon Saad-Falcon8c2ee6843ad2
diff --git a/src/openjarvis/engine/gemma_cpp.py b/src/openjarvis/engine/gemma_cpp.pyindex a235417b..4e2f8245 100644--- a/src/openjarvis/engine/gemma_cpp.py+++ b/src/openjarvis/engine/gemma_cpp.py@@ -14,6 +14,12 @@ from openjarvis.engine._base import InferenceEngine, estimate_prompt_tokenslogger = logging.getLogger(__name__)+def _import_pygemma():+ """Import and return the pygemma.Gemma class. Raises ImportError if unavailable."""+ from pygemma import Gemma+ return Gemma++@EngineRegistry.register("gemma_cpp")class GemmaCppEngine(InferenceEngine):"""gemma.cpp backend via pygemma pybind11 bindings (in-process, CPU)."""@@ -64,6 +70,35 @@ class GemmaCppEngine(InferenceEngine):parts.append("<start_of_turn>model\n")return "".join(parts)+ def _ensure_loaded(self) -> None:+ """Lazy model loading — called before inference."""+ if self._gemma is None:+ if not self._model_path:+ raise FileNotFoundError(+ "gemma.cpp model_path not configured. Download weights "+ "from Kaggle and set GEMMA_CPP_MODEL_PATH or configure "+ "[engine.gemma_cpp] in ~/.openjarvis/config.toml"+ )+ if not self._tokenizer_path:+ raise FileNotFoundError(+ "gemma.cpp tokenizer_path not configured. Set "+ "GEMMA_CPP_TOKENIZER_PATH or configure "+ "[engine.gemma_cpp] in ~/.openjarvis/config.toml"+ )+ Gemma = _import_pygemma()+ self._gemma = Gemma()+ self._gemma.load_model(+ self._tokenizer_path, self._model_path, self._model_type+ )++ def prepare(self, model: str) -> None:+ """Load model into memory."""+ self._ensure_loaded()++ def close(self) -> None:+ """Unload model and free memory."""+ self._gemma = None+def generate(self,messages: Sequence[Message],@@ -73,7 +108,31 @@ class GemmaCppEngine(InferenceEngine):max_tokens: int = 1024,**kwargs: Any,) -> Dict[str, Any]:- raise NotImplementedError # implemented in Task 4+ self._ensure_loaded()+ if model != self._model_type:+ logger.warning(+ "gemma_cpp: requested model %r but loaded model is %r; "+ "proceeding with loaded model",+ model, self._model_type,+ )+ prompt = self._messages_to_prompt(messages)+ try:+ raw = self._gemma.completion(prompt)+ except Exception as exc:+ raise RuntimeError(f"gemma.cpp inference failed: {exc}") from exc++ prompt_tokens = estimate_prompt_tokens(messages)+ completion_tokens = max(1, len(raw) // 4)+ return {+ "content": raw,+ "usage": {+ "prompt_tokens": prompt_tokens,+ "completion_tokens": completion_tokens,+ "total_tokens": prompt_tokens + completion_tokens,+ },+ "model": self._model_type,+ "finish_reason": "stop",+ }async def stream(self,@@ -84,8 +143,13 @@ class GemmaCppEngine(InferenceEngine):max_tokens: int = 1024,**kwargs: Any,) -> AsyncIterator[str]:- raise NotImplementedError # implemented in Task 4- yield "" # pragma: no cover+ self._ensure_loaded()+ prompt = self._messages_to_prompt(messages)+ try:+ raw = self._gemma.completion(prompt)+ except Exception as exc:+ raise RuntimeError(f"gemma.cpp inference failed: {exc}") from exc+ yield rawdef list_models(self) -> List[str]:raise NotImplementedError # implemented in Task 5@@ -93,11 +157,5 @@ class GemmaCppEngine(InferenceEngine):def health(self) -> bool:raise NotImplementedError # implemented in Task 5- def close(self) -> None:- pass # implemented in Task 4-- def prepare(self, model: str) -> None:- pass # implemented in Task 4-__all__ = ["GemmaCppEngine"]diff --git a/tests/engine/test_gemma_cpp.py b/tests/engine/test_gemma_cpp.pyindex 91f8bcfb..82d6b146 100644--- a/tests/engine/test_gemma_cpp.py+++ b/tests/engine/test_gemma_cpp.py@@ -90,3 +90,128 @@ class TestMessagesToPrompt:"Rule 1\n\nRule 2\n\nGo<end_of_turn>\n""<start_of_turn>model\n")+++from unittest.mock import MagicMock, patch+++class TestGemmaCppLifecycle:+ def _make_engine(self, **kwargs):+ from openjarvis.engine.gemma_cpp import GemmaCppEngine+ defaults = {+ "model_path": "/fake/model.sbs",+ "tokenizer_path": "/fake/tokenizer.spm",+ "model_type": "2b-it",+ }+ defaults.update(kwargs)+ return GemmaCppEngine(**defaults)++ @patch("openjarvis.engine.gemma_cpp._import_pygemma")+ def test_prepare_loads_model(self, mock_import) -> None:+ mock_gemma_cls = MagicMock()+ mock_import.return_value = mock_gemma_cls+ engine = self._make_engine()+ engine.prepare("2b-it")+ mock_gemma_cls.assert_called_once()+ mock_gemma_cls.return_value.load_model.assert_called_once_with(+ "/fake/tokenizer.spm", "/fake/model.sbs", "2b-it"+ )++ @patch("openjarvis.engine.gemma_cpp._import_pygemma")+ def test_prepare_idempotent(self, mock_import) -> None:+ mock_gemma_cls = MagicMock()+ mock_import.return_value = mock_gemma_cls+ engine = self._make_engine()+ engine.prepare("2b-it")+ engine.prepare("2b-it")+ # Only loaded once+ assert mock_gemma_cls.call_count == 1++ @patch("openjarvis.engine.gemma_cpp._import_pygemma")+ def test_close_unloads_model(self, mock_import) -> None:+ mock_gemma_cls = MagicMock()+ mock_import.return_value = mock_gemma_cls+ engine = self._make_engine()+ engine.prepare("2b-it")+ assert engine._gemma is not None+ engine.close()+ assert engine._gemma is None++ @patch("openjarvis.engine.gemma_cpp._import_pygemma")+ def test_generate_returns_dict(self, mock_import) -> None:+ mock_gemma_cls = MagicMock()+ mock_gemma_instance = MagicMock()+ mock_gemma_instance.completion.return_value = "The answer is 4."+ mock_gemma_cls.return_value = mock_gemma_instance+ mock_import.return_value = mock_gemma_cls++ engine = self._make_engine()+ result = engine.generate(+ [Message(role=Role.USER, content="What is 2+2?")],+ model="2b-it",+ )+ assert result["content"] == "The answer is 4."+ assert "usage" in result+ assert result["usage"]["prompt_tokens"] > 0+ assert result["usage"]["completion_tokens"] > 0+ assert result["usage"]["total_tokens"] > 0+ assert result["model"] == "2b-it"+ assert result["finish_reason"] == "stop"++ @patch("openjarvis.engine.gemma_cpp._import_pygemma")+ def test_generate_warns_on_model_mismatch(self, mock_import) -> None:+ mock_gemma_cls = MagicMock()+ mock_gemma_instance = MagicMock()+ mock_gemma_instance.completion.return_value = "ok"+ mock_gemma_cls.return_value = mock_gemma_instance+ mock_import.return_value = mock_gemma_cls++ engine = self._make_engine(model_type="2b-it")+ from openjarvis.engine import gemma_cpp as gc_mod+ with patch.object(gc_mod.logger, "warning") as mock_warn:+ engine.generate(+ [Message(role=Role.USER, content="Hi")],+ model="9b-it",+ )+ mock_warn.assert_called_once()++ @patch("openjarvis.engine.gemma_cpp._import_pygemma")+ def test_generate_wraps_runtime_error(self, mock_import) -> None:+ mock_gemma_cls = MagicMock()+ mock_gemma_instance = MagicMock()+ mock_gemma_instance.completion.side_effect = Exception("segfault")+ mock_gemma_cls.return_value = mock_gemma_instance+ mock_import.return_value = mock_gemma_cls++ engine = self._make_engine()+ with pytest.raises(RuntimeError, match="gemma.cpp inference failed"):+ engine.generate(+ [Message(role=Role.USER, content="Hi")],+ model="2b-it",+ )+++class TestGemmaCppStream:+ @patch("openjarvis.engine.gemma_cpp._import_pygemma")+ @pytest.mark.asyncio+ async def test_stream_yields_content(self, mock_import) -> None:+ mock_gemma_cls = MagicMock()+ mock_gemma_instance = MagicMock()+ mock_gemma_instance.completion.return_value = "streamed output"+ mock_gemma_cls.return_value = mock_gemma_instance+ mock_import.return_value = mock_gemma_cls++ from openjarvis.engine.gemma_cpp import GemmaCppEngine+ engine = GemmaCppEngine(+ model_path="/fake/model.sbs",+ tokenizer_path="/fake/tokenizer.spm",+ model_type="2b-it",+ )+ chunks = []+ async for chunk in engine.stream(+ [Message(role=Role.USER, content="Hi")],+ model="2b-it",+ ):+ chunks.append(chunk)+ assert len(chunks) == 1+ assert chunks[0] == "streamed output"