Personal AI, On Personal Devices
Python
fix(learning): persist compiled DSPy demos in skill overlays (#1042)
Elliot Slusky55eaff04e77e
diff --git a/src/openjarvis/learning/agents/dspy_optimizer.py b/src/openjarvis/learning/agents/dspy_optimizer.pyindex bbb338df..8967873d 100644--- a/src/openjarvis/learning/agents/dspy_optimizer.py+++ b/src/openjarvis/learning/agents/dspy_optimizer.py@@ -144,12 +144,15 @@ class DSPyAgentOptimizer:# Extract optimized parametersresult: Dict[str, Any] = {}- if hasattr(optimized_program, "generate") and hasattr(- optimized_program.generate, "demos"- ):+ # ChainOfThought wraps its predictor; BootstrapFewShot attaches demos+ # to that predictor, not to the ChainOfThought module.+ generate = getattr(optimized_program, "generate", None)+ predictor = getattr(generate, "predict", generate)+ demos = getattr(predictor, "demos", None)+ if demos is not None:result["few_shot_examples"] = [{"input": d.question, "output": d.answer}- for d in optimized_program.generate.demos+ for d in demosif hasattr(d, "question") and hasattr(d, "answer")]diff --git a/src/openjarvis/learning/agents/skill_optimizer.py b/src/openjarvis/learning/agents/skill_optimizer.pyindex bb495034..b5b5bf04 100644--- a/src/openjarvis/learning/agents/skill_optimizer.py+++ b/src/openjarvis/learning/agents/skill_optimizer.py@@ -220,15 +220,16 @@ class SkillOptimizer:return list(self._traces[:limit])try:- updates = optimizer.optimize(_BucketStore(skill_traces)) or {}+ optimization_result = optimizer.optimize(_BucketStore(skill_traces)) or {}except Exception as exc:LOGGER.warning("DSPyAgentOptimizer raised for '%s' (using empty output): %s",skill_name,exc,)- updates = {}+ optimization_result = {}+ updates = optimization_result.get("config_updates") or {}description = str(updates.get("system_prompt", "")) if updates else ""few_shot_raw = updates.get("few_shot_examples", []) if updates else []few_shot: List[Dict[str, str]] = []diff --git a/tests/learning/agents/test_dspy_optimizer.py b/tests/learning/agents/test_dspy_optimizer.pyindex 3904b75d..88eb95f7 100644--- a/tests/learning/agents/test_dspy_optimizer.py+++ b/tests/learning/agents/test_dspy_optimizer.py@@ -3,6 +3,7 @@from __future__ import annotationsimport time+from types import SimpleNamespacefrom unittest.mock import MagicMock, patch@@ -88,3 +89,58 @@ class TestDSPyOptimizerTraceConversion:result = optimizer.optimize(mock_store)assert result["status"] == "completed"assert "config_updates" in result++ def test_extracts_demos_from_compiled_chain_of_thought(self, monkeypatch) -> None:+ """DSPy stores compiled ChainOfThought demos on its inner predictor."""+ import openjarvis.learning.agents.dspy_optimizer as mod+ from openjarvis.core.config import DSPyOptimizerConfig++ class FakeExample:+ def __init__(self, *, question: str, answer: str) -> None:+ self.question = question+ self.answer = answer++ def with_inputs(self, *names: str):+ assert names == ("question",)+ return self++ class FakeModule:+ pass++ class FakeChainOfThought:+ def __init__(self, signature: str) -> None:+ assert signature == "question -> answer"++ class FakeBootstrapFewShot:+ def __init__(self, **kwargs) -> None:+ pass++ def compile(self, program, *, trainset):+ assert isinstance(program.generate, FakeChainOfThought)+ assert trainset[0].question == "How are you?"+ demos = [FakeExample(question="How are you?", answer="Well")]+ return SimpleNamespace(+ generate=SimpleNamespace(predict=SimpleNamespace(demos=demos))+ )++ fake_dspy = SimpleNamespace(+ Example=FakeExample,+ Module=FakeModule,+ ChainOfThought=FakeChainOfThought,+ BootstrapFewShot=FakeBootstrapFewShot,+ )+ monkeypatch.setattr(mod, "dspy", fake_dspy)+ monkeypatch.setattr(mod, "HAS_DSPY", True)++ optimizer = mod.DSPyAgentOptimizer(+ DSPyOptimizerConfig(optimizer="BootstrapFewShot", min_traces=1)+ )+ trace = SimpleNamespace(query="How are you?", result="Well", feedback=1.0)+ store = SimpleNamespace(list_traces=lambda **kwargs: [trace])++ result = optimizer.optimize(store)++ assert result["status"] == "completed"+ assert result["config_updates"]["few_shot_examples"] == [+ {"input": "How are you?", "output": "Well"}+ ]diff --git a/tests/learning/test_skill_optimizer.py b/tests/learning/test_skill_optimizer.pyindex 23c7323c..ea86ab20 100644--- a/tests/learning/test_skill_optimizer.py+++ b/tests/learning/test_skill_optimizer.py@@ -139,3 +139,44 @@ class TestSkillOptimizerOptimize:assert "An optimized description" in contentassert "hello" in contentassert "world" in content++ def test_dspy_compiled_demos_reach_skill_overlay(+ self, tmp_path: Path, monkeypatch: Any+ ) -> None:+ """Use DSPy's nested result rather than falling back to raw traces."""+ from openjarvis.learning.agents.dspy_optimizer import DSPyAgentOptimizer+ from openjarvis.learning.agents.skill_optimizer import SkillOptimizer+ from openjarvis.skills.overlay import SkillOverlayLoader++ traces = [_make_skill_trace("research-skill")]+ store = _FakeTraceStore(traces)+ manager = _make_manager_with_skill("research-skill", tmp_path)++ def fake_optimize(self, trace_store):+ assert trace_store.list_traces(limit=100)[0] is traces[0]+ return {+ "status": "completed",+ "config_updates": {+ "system_prompt": "Optimized skill description",+ "few_shot_examples": [+ {+ "input": "synthesized question",+ "output": "synthesized answer",+ }+ ],+ },+ }++ monkeypatch.setattr(DSPyAgentOptimizer, "optimize", fake_optimize)+ overlay_dir = tmp_path / "overlays"+ result = SkillOptimizer(min_traces_per_skill=1).optimize(+ store, manager, overlay_dir=overlay_dir+ )++ assert result["research-skill"].status == "optimized"+ overlay = SkillOverlayLoader(overlay_dir).load("research-skill")+ assert overlay is not None+ assert overlay.description == "Optimized skill description"+ assert overlay.few_shot == [+ {"input": "synthesized question", "output": "synthesized answer"}+ ]