Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

add image support in JSON mode (#7968)

Francesco Laiti4b07ae5a9f98
diff --git a/dspy/adapters/json_adapter.py b/dspy/adapters/json_adapter.py
index 7567e7876..2d1288f4f 100644
--- a/dspy/adapters/json_adapter.py
+++ b/dspy/adapters/json_adapter.py
@@ -13,7 +13,7 @@ from pydantic import create_model
from pydantic.fields import FieldInfo
from dspy.adapters.base import Adapter
-from dspy.adapters.types.image import Image
+from dspy.adapters.types.image import try_expand_image_tags
from dspy.adapters.types.history import History
from dspy.adapters.utils import format_field_value, get_annotation_name, parse_value, serialize_for_json
from dspy.clients.lm import LM
@@ -42,11 +42,11 @@ class JSONAdapter(Adapter):
try:
response_format = _get_structured_outputs_response_format(signature)
outputs = lm(**inputs, **lm_kwargs, response_format=response_format)
- except Exception:
+ except Exception as e:
logger.debug(
- "Failed to obtain response using signature-based structured outputs"
- " response format: Falling back to default 'json_object' response format."
- " Exception: {e}"
+ f"Failed to obtain response using signature-based structured outputs"
+ f" response format: Falling back to default 'json_object' response format."
+ f" Exception: {e}"
)
outputs = lm(**inputs, **lm_kwargs, response_format={"type": "json_object"})
else:
@@ -92,6 +92,7 @@ class JSONAdapter(Adapter):
else:
messages.append(self.format_turn(signature, inputs, role="user"))
+ messages = try_expand_image_tags(messages)
return messages
def parse(self, signature: Type[Signature], completion: str) -> dict[str, Any]:
@@ -116,7 +117,6 @@ class JSONAdapter(Adapter):
for field_name, field_info in signature.fields.items()
if field_name in values
}
-
return format_fields(role=role, fields_with_values=fields_with_values)
def format_turn(self, signature: Type[Signature], values, role: str, incomplete: bool = False, is_conversation_history: bool = False) -> dict[str, Any]:
@@ -127,34 +127,16 @@ class JSONAdapter(Adapter):
raise NotImplementedError
-def _format_field_value(field_info: FieldInfo, value: Any) -> str:
- """
- Formats the value of the specified field according to the field's DSPy type (input or output),
- annotation (e.g. str, int, etc.), and the type of the value itself.
-
- Args:
- field_info: Information about the field, including its DSPy field type and annotation.
- value: The value of the field.
-
- Returns:
- The formatted value of the field, represented as a string.
- """
- # TODO: Wasnt this easy to fix?
- if field_info.annotation is Image:
- raise NotImplementedError("Images are not yet supported in JSON mode.")
-
- return format_field_value(field_info=field_info, value=value)
-
-
def format_fields(role: str, fields_with_values: Dict[FieldInfoWithName, Any]) -> str:
"""
Formats the values of the specified fields according to the field's DSPy type (input or output),
annotation (e.g. str, int, etc.), and the type of the value itself. Joins the formatted values
- into a single string, which is is a multiline string if there are multiple fields.
+ into a single string, which is a multiline string if there are multiple fields.
Args:
- fields_with_values: A dictionary mapping information about a field to its corresponding
- value.
+ role: The role of the message ('user' or 'assistant')
+ fields_with_values: A dictionary mapping information about a field to its corresponding value.
+
Returns:
The joined formatted values of the fields, represented as a string.
"""
@@ -166,7 +148,7 @@ def format_fields(role: str, fields_with_values: Dict[FieldInfoWithName, Any]) -
output = []
for field, field_value in fields_with_values.items():
- formatted_field_value = _format_field_value(field_info=field.info, value=field_value)
+ formatted_field_value = format_field_value(field_info=field.info, value=field_value)
output.append(f"[[ ## {field.name} ## ]]\n{formatted_field_value}")
return "\n\n".join(output).strip()
@@ -175,7 +157,7 @@ def format_fields(role: str, fields_with_values: Dict[FieldInfoWithName, Any]) -
def format_turn(
signature: SignatureMeta,
values: Dict[str, Any],
- role,
+ role: str,
incomplete=False,
is_conversation_history=False,
) -> Dict[str, str]: