Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Add experimental annotation (#8712)

TomuHirata8c678ebbab47
diff --git a/dspy/teleprompt/gepa/gepa.py b/dspy/teleprompt/gepa/gepa.py
index 1f63f17ce..fd39e9be3 100644
--- a/dspy/teleprompt/gepa/gepa.py
+++ b/dspy/teleprompt/gepa/gepa.py
@@ -11,6 +11,7 @@ from dspy.clients.lm import LM
from dspy.primitives import Example, Module, Prediction
from dspy.teleprompt.gepa.gepa_utils import DspyAdapter, DSPyTrace, PredictorFeedbackFn, ScoreWithFeedback
from dspy.teleprompt.teleprompt import Teleprompter
+from dspy.utils.annotation import experimental
logger = logging.getLogger(__name__)
@@ -20,6 +21,7 @@ AUTO_RUN_SETTINGS = {
"heavy": {"n": 18},
}
+@experimental(version="3.0.0")
class GEPAFeedbackMetric(Protocol):
def __call__(
gold: Example,
@@ -49,6 +51,7 @@ class GEPAFeedbackMetric(Protocol):
"""
...
+@experimental(version="3.0.0")
@dataclass(frozen=True)
class DspyGEPAResult:
"""
@@ -140,6 +143,7 @@ class DspyGEPAResult:
seed=gepa_result.seed,
)
+@experimental(version="3.0.0")
class GEPA(Teleprompter):
"""
GEPA is an evolutionary optimizer, which uses reflection to evolve text components
diff --git a/dspy/utils/__init__.py b/dspy/utils/__init__.py
index 0c53086f6..8f5e9cf5e 100644
--- a/dspy/utils/__init__.py
+++ b/dspy/utils/__init__.py
@@ -4,6 +4,7 @@ import requests
from dspy.streaming.messages import StatusMessage, StatusMessageProvider
from dspy.utils import exceptions
+from dspy.utils.annotation import experimental
from dspy.utils.callback import BaseCallback, with_callbacks
from dspy.utils.dummies import DummyLM, DummyVectorizer, dummy_rm
from dspy.utils.inspect_history import pretty_print_history
@@ -30,6 +31,7 @@ __all__ = [
"DummyLM",
"DummyVectorizer",
"dummy_rm",
+ "experimental",
"StatusMessage",
"StatusMessageProvider",
"pretty_print_history",
diff --git a/dspy/utils/annotation.py b/dspy/utils/annotation.py
new file mode 100644
index 000000000..4b12c142d
--- /dev/null
+++ b/dspy/utils/annotation.py
@@ -0,0 +1,88 @@
+import inspect
+import re
+import types
+from typing import Callable, ParamSpec, TypeVar, overload
+
+P = ParamSpec("P")
+R = TypeVar("R")
+
+@overload
+def experimental(f: Callable[P, R], version: str | None = None) -> Callable[P, R]: ...
+
+@overload
+def experimental(f: None = None, version: str | None = None) -> Callable[[Callable[P, R]], Callable[P, R]]: ...
+
+
+def experimental(
+ f: Callable[P, R] | None = None,
+ version: str | None = None,
+) -> Callable[[Callable[P, R]], Callable[P, R]]:
+ """Decorator / decorator creator for marking APIs experimental in the docstring.
+
+ Args:
+ f: The function to be decorated.
+ version: The version in which the API was introduced as experimental.
+ The version is used to determine whether the API should be considered
+ as stable or not when releasing a new version of DSPy.
+
+ Returns:
+ A decorator that adds a note to the docstring of the decorated API.
+ """
+ if f:
+ return _experimental(f, version)
+ else:
+ def decorator(f: Callable[P, R]) -> Callable[P, R]:
+ return _experimental(f, version)
+ return decorator
+
+
+def _experimental(api: Callable[P, R], version: str | None = None) -> Callable[P, R]:
+ """Add experimental notice to the API's docstring."""
+ if inspect.isclass(api):
+ api_type = "class"
+ elif inspect.isfunction(api):
+ api_type = "function"
+ elif isinstance(api, property):
+ api_type = "property"
+ elif isinstance(api, types.MethodType):
+ api_type = "method"
+ else:
+ api_type = str(type(api))
+
+ indent = _get_min_indent_of_docstring(api.__doc__) if api.__doc__ else ""
+
+ version_text = f" (introduced in v{version})" if version else ""
+ notice = (
+ indent + f"Experimental: This {api_type} may change or "
+ f"be removed in a future release without warning{version_text}."
+ )
+
+ if api_type == "property":
+ api.__doc__ = api.__doc__ + "\n\n" + notice if api.__doc__ else notice
+ else:
+ if api.__doc__:
+ api.__doc__ = notice + "\n\n" + api.__doc__
+ else:
+ api.__doc__ = notice
+ return api
+
+
+def _get_min_indent_of_docstring(docstring_str: str) -> str:
+ """
+ Get the minimum indentation string of a docstring, based on the assumption
+ that the closing triple quote for multiline comments must be on a new line.
+ Note that based on ruff rule D209, the closing triple quote for multiline
+ comments must be on a new line.
+
+ Args:
+ docstring_str: string with docstring
+
+ Returns:
+ Whitespace corresponding to the indent of a docstring.
+ """
+
+ if not docstring_str or "\n" not in docstring_str:
+ return ""
+
+ match = re.match(r"^\s*", docstring_str.rsplit("\n", 1)[-1])
+ return match.group() if match else ""
diff --git a/tests/utils/test_annotation.py b/tests/utils/test_annotation.py
new file mode 100644
index 000000000..215a1259c
--- /dev/null
+++ b/tests/utils/test_annotation.py
@@ -0,0 +1,92 @@
+from dspy.utils.annotation import experimental
+
+
+def test_experimental_decorator_on_function():
+ @experimental
+ def test_function():
+ """A test function."""
+ return "test"
+
+ assert "Experimental: This function may change or be removed in a future release without warning." in test_function.__doc__
+ assert "A test function." in test_function.__doc__
+ assert test_function() == "test"
+
+
+def test_experimental_decorator_on_function_with_version():
+ @experimental(version="3.1.0")
+ def test_function():
+ """A test function with version."""
+ return "versioned"
+
+ assert "introduced in v3.1.0" in test_function.__doc__
+ assert "Experimental: This function may change or be removed in a future release without warning (introduced in v3.1.0)." in test_function.__doc__
+ assert "A test function with version." in test_function.__doc__
+ assert test_function() == "versioned"
+
+
+def test_experimental_decorator_on_class():
+ @experimental
+ class TestClass:
+ """A test class."""
+
+ def method(self):
+ return "method"
+
+ assert "Experimental: This class may change or be removed in a future release without warning." in TestClass.__doc__
+ assert "A test class." in TestClass.__doc__
+
+ instance = TestClass()
+ assert instance.method() == "method"
+
+
+def test_experimental_decorator_on_class_with_version():
+ @experimental(version="2.5.0")
+ class TestClass:
+ """A test class with version."""
+ pass
+
+ assert "introduced in v2.5.0" in TestClass.__doc__
+ assert "Experimental: This class may change or be removed in a future release without warning (introduced in v2.5.0)." in TestClass.__doc__
+ assert "A test class with version." in TestClass.__doc__
+
+
+def test_experimental_decorator_without_docstring():
+ @experimental
+ def test_function():
+ return "no_doc"
+
+ assert test_function.__doc__ == "Experimental: This function may change or be removed in a future release without warning."
+ assert test_function() == "no_doc"
+
+
+def test_experimental_decorator_without_docstring_with_version():
+ @experimental(version="1.0.0")
+ def test_function():
+ return "no_doc_version"
+
+ assert test_function.__doc__ == "Experimental: This function may change or be removed in a future release without warning (introduced in v1.0.0)."
+ assert test_function() == "no_doc_version"
+
+
+def test_experimental_decorator_with_callable_syntax():
+ def test_function():
+ """A test function."""
+ return "callable"
+
+ decorated = experimental(test_function)
+
+ assert "Experimental:" in decorated.__doc__
+ assert "A test function." in decorated.__doc__
+ assert decorated() == "callable"
+
+
+def test_experimental_decorator_with_version_callable_syntax():
+ def test_function():
+ """A test function."""
+ return "callable_version"
+
+ decorated = experimental(test_function, version="4.0.0")
+
+ assert "introduced in v4.0.0" in decorated.__doc__
+ assert "Experimental:" in decorated.__doc__
+ assert decorated() == "callable_version"