Skip to content

Personal AI, On Personal Devices

Python

fix(learning): persist compiled DSPy demos in skill overlays (#1042)

Elliot Slusky55eaff04e77e
diff --git a/src/openjarvis/learning/agents/dspy_optimizer.py b/src/openjarvis/learning/agents/dspy_optimizer.py
index bbb338df..8967873d 100644
--- a/src/openjarvis/learning/agents/dspy_optimizer.py
+++ b/src/openjarvis/learning/agents/dspy_optimizer.py
@@ -144,12 +144,15 @@ class DSPyAgentOptimizer:
# Extract optimized parameters
result: Dict[str, Any] = {}
- if hasattr(optimized_program, "generate") and hasattr(
- optimized_program.generate, "demos"
- ):
+ # ChainOfThought wraps its predictor; BootstrapFewShot attaches demos
+ # to that predictor, not to the ChainOfThought module.
+ generate = getattr(optimized_program, "generate", None)
+ predictor = getattr(generate, "predict", generate)
+ demos = getattr(predictor, "demos", None)
+ if demos is not None:
result["few_shot_examples"] = [
{"input": d.question, "output": d.answer}
- for d in optimized_program.generate.demos
+ for d in demos
if hasattr(d, "question") and hasattr(d, "answer")
]
diff --git a/src/openjarvis/learning/agents/skill_optimizer.py b/src/openjarvis/learning/agents/skill_optimizer.py
index bb495034..b5b5bf04 100644
--- a/src/openjarvis/learning/agents/skill_optimizer.py
+++ b/src/openjarvis/learning/agents/skill_optimizer.py
@@ -220,15 +220,16 @@ class SkillOptimizer:
return list(self._traces[:limit])
try:
- updates = optimizer.optimize(_BucketStore(skill_traces)) or {}
+ optimization_result = optimizer.optimize(_BucketStore(skill_traces)) or {}
except Exception as exc:
LOGGER.warning(
"DSPyAgentOptimizer raised for '%s' (using empty output): %s",
skill_name,
exc,
)
- updates = {}
+ optimization_result = {}
+ updates = optimization_result.get("config_updates") or {}
description = str(updates.get("system_prompt", "")) if updates else ""
few_shot_raw = updates.get("few_shot_examples", []) if updates else []
few_shot: List[Dict[str, str]] = []
diff --git a/tests/learning/agents/test_dspy_optimizer.py b/tests/learning/agents/test_dspy_optimizer.py
index 3904b75d..88eb95f7 100644
--- a/tests/learning/agents/test_dspy_optimizer.py
+++ b/tests/learning/agents/test_dspy_optimizer.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import time
+from types import SimpleNamespace
from unittest.mock import MagicMock, patch
@@ -88,3 +89,58 @@ class TestDSPyOptimizerTraceConversion:
result = optimizer.optimize(mock_store)
assert result["status"] == "completed"
assert "config_updates" in result
+
+ def test_extracts_demos_from_compiled_chain_of_thought(self, monkeypatch) -> None:
+ """DSPy stores compiled ChainOfThought demos on its inner predictor."""
+ import openjarvis.learning.agents.dspy_optimizer as mod
+ from openjarvis.core.config import DSPyOptimizerConfig
+
+ class FakeExample:
+ def __init__(self, *, question: str, answer: str) -> None:
+ self.question = question
+ self.answer = answer
+
+ def with_inputs(self, *names: str):
+ assert names == ("question",)
+ return self
+
+ class FakeModule:
+ pass
+
+ class FakeChainOfThought:
+ def __init__(self, signature: str) -> None:
+ assert signature == "question -> answer"
+
+ class FakeBootstrapFewShot:
+ def __init__(self, **kwargs) -> None:
+ pass
+
+ def compile(self, program, *, trainset):
+ assert isinstance(program.generate, FakeChainOfThought)
+ assert trainset[0].question == "How are you?"
+ demos = [FakeExample(question="How are you?", answer="Well")]
+ return SimpleNamespace(
+ generate=SimpleNamespace(predict=SimpleNamespace(demos=demos))
+ )
+
+ fake_dspy = SimpleNamespace(
+ Example=FakeExample,
+ Module=FakeModule,
+ ChainOfThought=FakeChainOfThought,
+ BootstrapFewShot=FakeBootstrapFewShot,
+ )
+ monkeypatch.setattr(mod, "dspy", fake_dspy)
+ monkeypatch.setattr(mod, "HAS_DSPY", True)
+
+ optimizer = mod.DSPyAgentOptimizer(
+ DSPyOptimizerConfig(optimizer="BootstrapFewShot", min_traces=1)
+ )
+ trace = SimpleNamespace(query="How are you?", result="Well", feedback=1.0)
+ store = SimpleNamespace(list_traces=lambda **kwargs: [trace])
+
+ result = optimizer.optimize(store)
+
+ assert result["status"] == "completed"
+ assert result["config_updates"]["few_shot_examples"] == [
+ {"input": "How are you?", "output": "Well"}
+ ]
diff --git a/tests/learning/test_skill_optimizer.py b/tests/learning/test_skill_optimizer.py
index 23c7323c..ea86ab20 100644
--- a/tests/learning/test_skill_optimizer.py
+++ b/tests/learning/test_skill_optimizer.py
@@ -139,3 +139,44 @@ class TestSkillOptimizerOptimize:
assert "An optimized description" in content
assert "hello" in content
assert "world" in content
+
+ def test_dspy_compiled_demos_reach_skill_overlay(
+ self, tmp_path: Path, monkeypatch: Any
+ ) -> None:
+ """Use DSPy's nested result rather than falling back to raw traces."""
+ from openjarvis.learning.agents.dspy_optimizer import DSPyAgentOptimizer
+ from openjarvis.learning.agents.skill_optimizer import SkillOptimizer
+ from openjarvis.skills.overlay import SkillOverlayLoader
+
+ traces = [_make_skill_trace("research-skill")]
+ store = _FakeTraceStore(traces)
+ manager = _make_manager_with_skill("research-skill", tmp_path)
+
+ def fake_optimize(self, trace_store):
+ assert trace_store.list_traces(limit=100)[0] is traces[0]
+ return {
+ "status": "completed",
+ "config_updates": {
+ "system_prompt": "Optimized skill description",
+ "few_shot_examples": [
+ {
+ "input": "synthesized question",
+ "output": "synthesized answer",
+ }
+ ],
+ },
+ }
+
+ monkeypatch.setattr(DSPyAgentOptimizer, "optimize", fake_optimize)
+ overlay_dir = tmp_path / "overlays"
+ result = SkillOptimizer(min_traces_per_skill=1).optimize(
+ store, manager, overlay_dir=overlay_dir
+ )
+
+ assert result["research-skill"].status == "optimized"
+ overlay = SkillOverlayLoader(overlay_dir).load("research-skill")
+ assert overlay is not None
+ assert overlay.description == "Optimized skill description"
+ assert overlay.few_shot == [
+ {"input": "synthesized question", "output": "synthesized answer"}
+ ]