Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Parse doubly-encoded base type in json.parse (#8814)

TomuHirata004d232c0519
diff --git a/dspy/adapters/types/base_type.py b/dspy/adapters/types/base_type.py
index 1a47eb2b4..8f82b87b7 100644
--- a/dspy/adapters/types/base_type.py
+++ b/dspy/adapters/types/base_type.py
@@ -64,7 +64,9 @@ class Type(pydantic.BaseModel):
def serialize_model(self):
formatted = self.format()
if isinstance(formatted, list):
- return f"{CUSTOM_TYPE_START_IDENTIFIER}{formatted}{CUSTOM_TYPE_END_IDENTIFIER}"
+ return (
+ f"{CUSTOM_TYPE_START_IDENTIFIER}{json.dumps(formatted, ensure_ascii=False)}{CUSTOM_TYPE_END_IDENTIFIER}"
+ )
return formatted
@@ -114,19 +116,21 @@ def split_message_content_for_custom_types(messages: list[dict[str, Any]]) -> li
# Parse the JSON inside the block
custom_type_content = match.group(1).strip()
- try:
+ parsed = None
+
+ for parse_fn in [json.loads, _parse_doubly_quoted_json, json_repair.loads]:
try:
- # Replace single quotes with double quotes to make it valid JSON
- parsed = json.loads(custom_type_content.replace("'", '"'))
+ parsed = parse_fn(custom_type_content)
+ break
except json.JSONDecodeError:
- parsed = json_repair.loads(custom_type_content)
+ continue
+
+ if parsed:
for custom_type_content in parsed:
result.append(custom_type_content)
-
- except json.JSONDecodeError:
+ else:
# fallback to raw string if it's not valid JSON
- parsed = {"type": "text", "text": custom_type_content}
- result.append(parsed)
+ result.append({"type": "text", "text": custom_type_content})
last_end = end
@@ -141,3 +145,11 @@ def split_message_content_for_custom_types(messages: list[dict[str, Any]]) -> li
message["content"] = result
return messages
+
+
+def _parse_doubly_quoted_json(json_str: str) -> Any:
+ """
+ Parse a doubly quoted JSON string into a Python dict.
+ `dspy.Type` can be json-encoded twice if included in either list or dict, e.g., `list[dspy.experimental.Document]`
+ """
+ return json.loads(json.loads(f'"{json_str}"'))
diff --git a/tests/adapters/test_json_adapter.py b/tests/adapters/test_json_adapter.py
index 7d703b5e9..c6ba8b238 100644
--- a/tests/adapters/test_json_adapter.py
+++ b/tests/adapters/test_json_adapter.py
@@ -316,6 +316,29 @@ def test_json_adapter_formats_image_with_nested_images():
assert expected_image3_content in messages[1]["content"]
+def test_json_adapter_formats_with_nested_documents():
+ class DocumentWrapper(pydantic.BaseModel):
+ documents: list[dspy.experimental.Document]
+
+ class MySignature(dspy.Signature):
+ document: DocumentWrapper = dspy.InputField()
+ text: str = dspy.OutputField()
+
+ doc1 = dspy.experimental.Document(data="Hello, world!")
+ doc2 = dspy.experimental.Document(data="Hello, world 2!")
+
+ document_wrapper = DocumentWrapper(documents=[doc1, doc2])
+
+ adapter = dspy.JSONAdapter()
+ messages = adapter.format(MySignature, [], {"document": document_wrapper})
+
+ expected_doc1_content = {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "Hello, world!"}, "citations": {"enabled": True}}
+ expected_doc2_content = {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "Hello, world 2!"}, "citations": {"enabled": True}}
+
+ assert expected_doc1_content in messages[1]["content"]
+ assert expected_doc2_content in messages[1]["content"]
+
+
def test_json_adapter_formats_image_with_few_shot_examples_with_nested_images():
class ImageWrapper(pydantic.BaseModel):
images: list[dspy.Image]
diff --git a/tests/signatures/test_adapter_image.py b/tests/signatures/test_adapter_image.py
index 8537858ae..07c8154e7 100644
--- a/tests/signatures/test_adapter_image.py
+++ b/tests/signatures/test_adapter_image.py
@@ -451,14 +451,14 @@ def test_image_repr():
url_image = dspy.Image("https://example.com/dog.jpg")
assert str(url_image) == (
"<<CUSTOM-TYPE-START-IDENTIFIER>>"
- "[{'type': 'image_url', 'image_url': {'url': 'https://example.com/dog.jpg'}}]"
+ '[{"type": "image_url", "image_url": {"url": "https://example.com/dog.jpg"}}]'
"<<CUSTOM-TYPE-END-IDENTIFIER>>"
)
assert repr(url_image) == "Image(url='https://example.com/dog.jpg')"
sample_pil = PILImage.new("RGB", (60, 30), color="red")
pil_image = dspy.Image(sample_pil)
- assert str(pil_image).startswith("<<CUSTOM-TYPE-START-IDENTIFIER>>[{'type': 'image_url',")
+ assert str(pil_image).startswith('<<CUSTOM-TYPE-START-IDENTIFIER>>[{"type": "image_url",')
assert str(pil_image).endswith("<<CUSTOM-TYPE-END-IDENTIFIER>>")
assert "base64" in str(pil_image)