Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

feat(File): add File type for handling file data (#9014)

TomuHiratac542bb64a7bb
diff --git a/dspy/__init__.py b/dspy/__init__.py
index ea4c75a86..01aad3675 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -6,7 +6,7 @@ from dspy.teleprompt import *
from dspy.evaluate import Evaluate # isort: skip
from dspy.clients import * # isort: skip
-from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, XMLAdapter, TwoStepAdapter, Image, Audio, History, Type, Tool, ToolCalls, Code # isort: skip
+from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, XMLAdapter, TwoStepAdapter, Image, Audio, File, History, Type, Tool, ToolCalls, Code # isort: skip
from dspy.utils.logging_utils import configure_dspy_loggers, disable_logging, enable_logging
from dspy.utils.asyncify import asyncify
from dspy.utils.syncify import syncify
diff --git a/dspy/adapters/__init__.py b/dspy/adapters/__init__.py
index 1dea6da47..fded25398 100644
--- a/dspy/adapters/__init__.py
+++ b/dspy/adapters/__init__.py
@@ -2,7 +2,7 @@ from dspy.adapters.base import Adapter
from dspy.adapters.chat_adapter import ChatAdapter
from dspy.adapters.json_adapter import JSONAdapter
from dspy.adapters.two_step_adapter import TwoStepAdapter
-from dspy.adapters.types import Audio, Code, History, Image, Tool, ToolCalls, Type
+from dspy.adapters.types import Audio, Code, File, History, Image, Tool, ToolCalls, Type
from dspy.adapters.xml_adapter import XMLAdapter
__all__ = [
@@ -12,6 +12,7 @@ __all__ = [
"History",
"Image",
"Audio",
+ "File",
"Code",
"JSONAdapter",
"XMLAdapter",
diff --git a/dspy/adapters/types/__init__.py b/dspy/adapters/types/__init__.py
index 11b9faee1..eb1481c86 100644
--- a/dspy/adapters/types/__init__.py
+++ b/dspy/adapters/types/__init__.py
@@ -1,8 +1,9 @@
from dspy.adapters.types.audio import Audio
from dspy.adapters.types.base_type import Type
from dspy.adapters.types.code import Code
+from dspy.adapters.types.file import File
from dspy.adapters.types.history import History
from dspy.adapters.types.image import Image
from dspy.adapters.types.tool import Tool, ToolCalls
-__all__ = ["History", "Image", "Audio", "Type", "Tool", "ToolCalls", "Code"]
+__all__ = ["History", "Image", "Audio", "File", "Type", "Tool", "ToolCalls", "Code"]
diff --git a/dspy/adapters/types/file.py b/dspy/adapters/types/file.py
new file mode 100644
index 000000000..34bc2a011
--- /dev/null
+++ b/dspy/adapters/types/file.py
@@ -0,0 +1,178 @@
+import base64
+import mimetypes
+import os
+from typing import Any
+
+import pydantic
+
+from dspy.adapters.types.base_type import Type
+
+
+class File(Type):
+ """A file input type for DSPy.
+ See https://platform.openai.com/docs/api-reference/chat/create#chat_create-messages-user_message-content-array_of_content_parts-file_content_part-file for specification.
+
+ The file_data field should be a data URI with the format:
+ data:<mime_type>;base64,<base64_encoded_data>
+
+ Example:
+ ```python
+ import dspy
+
+ class QA(dspy.Signature):
+ file: dspy.File = dspy.InputField()
+ summary = dspy.OutputField()
+ program = dspy.Predict(QA)
+ result = program(file=dspy.File.from_path("./research.pdf"))
+ print(result.summary)
+ ```
+ """
+
+ file_data: str | None = None
+ file_id: str | None = None
+ filename: str | None = None
+
+ model_config = pydantic.ConfigDict(
+ frozen=True,
+ str_strip_whitespace=True,
+ validate_assignment=True,
+ extra="forbid",
+ )
+
+ @pydantic.model_validator(mode="before")
+ @classmethod
+ def validate_input(cls, values: Any) -> Any:
+ if isinstance(values, cls):
+ return {
+ "file_data": values.file_data,
+ "file_id": values.file_id,
+ "filename": values.filename,
+ }
+
+ if isinstance(values, dict):
+ if "file_data" in values or "file_id" in values or "filename" in values:
+ return values
+ raise ValueError("Value of `dspy.File` must contain at least one of: file_data, file_id, or filename")
+
+ return encode_file_to_dict(values)
+
+ def format(self) -> list[dict[str, Any]]:
+ try:
+ file_dict = {}
+ if self.file_data:
+ file_dict["file_data"] = self.file_data
+ if self.file_id:
+ file_dict["file_id"] = self.file_id
+ if self.filename:
+ file_dict["filename"] = self.filename
+
+ return [{"type": "file", "file": file_dict}]
+ except Exception as e:
+ raise ValueError(f"Failed to format file for DSPy: {e}")
+
+ def __str__(self):
+ return self.serialize_model()
+
+ def __repr__(self):
+ parts = []
+ if self.file_data is not None:
+ if self.file_data.startswith("data:"):
+ # file data has "data:text/plain;base64,..." format
+ mime_type = self.file_data.split(";")[0].split(":")[1]
+ len_data = len(self.file_data.split("base64,")[1]) if "base64," in self.file_data else len(self.file_data)
+ parts.append(f"file_data=<DATA_URI({mime_type}, {len_data} chars)>")
+ else:
+ len_data = len(self.file_data)
+ parts.append(f"file_data=<DATA({len_data} chars)>")
+ if self.file_id is not None:
+ parts.append(f"file_id='{self.file_id}'")
+ if self.filename is not None:
+ parts.append(f"filename='{self.filename}'")
+ return f"File({', '.join(parts)})"
+
+ @classmethod
+ def from_path(cls, file_path: str, filename: str | None = None, mime_type: str | None = None) -> "File":
+ """Create a File from a local file path.
+
+ Args:
+ file_path: Path to the file to read
+ filename: Optional filename to use (defaults to basename of path)
+ mime_type: Optional MIME type (defaults to auto-detection from file extension)
+ """
+ if not os.path.isfile(file_path):
+ raise ValueError(f"File not found: {file_path}")
+
+ with open(file_path, "rb") as f:
+ file_bytes = f.read()
+
+ if filename is None:
+ filename = os.path.basename(file_path)
+
+ if mime_type is None:
+ mime_type, _ = mimetypes.guess_type(file_path)
+ if mime_type is None:
+ mime_type = "application/octet-stream"
+
+ encoded_data = base64.b64encode(file_bytes).decode("utf-8")
+ file_data = f"data:{mime_type};base64,{encoded_data}"
+
+ return cls(file_data=file_data, filename=filename)
+
+ @classmethod
+ def from_bytes(
+ cls, file_bytes: bytes, filename: str | None = None, mime_type: str = "application/octet-stream"
+ ) -> "File":
+ """Create a File from raw bytes.
+
+ Args:
+ file_bytes: Raw bytes of the file
+ filename: Optional filename
+ mime_type: MIME type (defaults to 'application/octet-stream')
+ """
+ encoded_data = base64.b64encode(file_bytes).decode("utf-8")
+ file_data = f"data:{mime_type};base64,{encoded_data}"
+ return cls(file_data=file_data, filename=filename)
+
+ @classmethod
+ def from_file_id(cls, file_id: str, filename: str | None = None) -> "File":
+ """Create a File from an uploaded file ID."""
+ return cls(file_id=file_id, filename=filename)
+
+
+def encode_file_to_dict(file_input: Any) -> dict:
+ """
+ Encode various file inputs to a dict with file_data, file_id, and/or filename.
+
+ Args:
+ file_input: Can be a file path (str), bytes, or File instance.
+
+ Returns:
+ dict: A dictionary with file_data, file_id, and/or filename keys.
+ """
+ if isinstance(file_input, File):
+ result = {}
+ if file_input.file_data is not None:
+ result["file_data"] = file_input.file_data
+ if file_input.file_id is not None:
+ result["file_id"] = file_input.file_id
+ if file_input.filename is not None:
+ result["filename"] = file_input.filename
+ return result
+
+ elif isinstance(file_input, str):
+ if os.path.isfile(file_input):
+ file_obj = File.from_path(file_input)
+ else:
+ raise ValueError(f"Unrecognized file string: {file_input}; must be a valid file path")
+
+ return {
+ "file_data": file_obj.file_data,
+ "filename": file_obj.filename,
+ }
+
+ elif isinstance(file_input, bytes):
+ file_obj = File.from_bytes(file_input)
+ return {"file_data": file_obj.file_data}
+
+ else:
+ raise ValueError(f"Unsupported file input type: {type(file_input)}")
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py
index f9fc648ca..81d968703 100644
--- a/dspy/clients/lm.py
+++ b/dspy/clients/lm.py
@@ -476,6 +476,11 @@ async def alitellm_responses_completion(request: dict[str, Any], num_retries: in
def _convert_chat_request_to_responses_request(request: dict[str, Any]):
+ """
+ Convert a chat request to a responses request
+ See https://platform.openai.com/docs/api-reference/responses/create for the responses API specification.
+ Also see https://platform.openai.com/docs/api-reference/chat/create for the chat API specification.
+ """
request = dict(request)
if "messages" in request:
content_blocks = []
@@ -525,6 +530,14 @@ def _convert_content_item_to_responses_format(item: dict[str, Any]) -> dict[str,
"type": "input_text",
"text": item.get("text", ""),
}
+ elif item.get("type") == "file":
+ file = item.get("file", {})
+ return {
+ "type": "input_file",
+ "file_data": file.get("file_data"),
+ "filename": file.get("filename"),
+ "file_id": file.get("file_id"),
+ }
# For other items, return as-is
return item
diff --git a/dspy/utils/inspect_history.py b/dspy/utils/inspect_history.py
index 556913a74..07934157f 100644
--- a/dspy/utils/inspect_history.py
+++ b/dspy/utils/inspect_history.py
@@ -46,6 +46,13 @@ def pretty_print_history(history, n: int = 1):
len_audio = len(c["input_audio"]["data"])
audio_str = f"<audio format='{audio_format}' base64-encoded, length={len_audio}>"
print(_blue(audio_str.strip()))
+ elif c["type"] == "file" or c["type"] == "input_file":
+ file = c.get("file", c.get("input_file", {}))
+ filename = file.get("filename", "")
+ file_id = file.get("file_id", "")
+ file_data = file.get("file_data", "")
+ file_str = f"<file: name:{filename}, id:{file_id}, data_length:{len(file_data)}>"
+ print(_blue(file_str.strip()))
print("\n")
if isinstance(outputs[0], dict):
diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.py
index 64e76ecd9..3a228286f 100644
--- a/tests/clients/test_lm.py
+++ b/tests/clients/test_lm.py
@@ -716,6 +716,103 @@ def test_responses_api_converts_images_correctly():
assert content[0]["image_url"] == "https://example.com/image.jpg"
+def test_responses_api_converts_files_correctly():
+ from dspy.clients.lm import _convert_chat_request_to_responses_request
+
+ # Test with file data (base64 encoded)
+ request_with_file = {
+ "model": "openai/gpt-5-mini",
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "Analyze this file"},
+ {
+ "type": "file",
+ "file": {
+ "file_data": "data:text/plain;base64,SGVsbG8gV29ybGQ=",
+ "filename": "test.txt",
+ }
+ }
+ ]
+ }
+ ]
+ }
+
+ result = _convert_chat_request_to_responses_request(request_with_file)
+
+ assert "input" in result
+ assert len(result["input"]) == 1
+ assert result["input"][0]["role"] == "user"
+
+ content = result["input"][0]["content"]
+ assert len(content) == 2
+
+ # First item should be text converted to input_text format
+ assert content[0]["type"] == "input_text"
+ assert content[0]["text"] == "Analyze this file"
+
+ # Second item should be converted to input_file format
+ assert content[1]["type"] == "input_file"
+ assert content[1]["file_data"] == "data:text/plain;base64,SGVsbG8gV29ybGQ="
+ assert content[1]["filename"] == "test.txt"
+
+ # Test with file_id
+ request_with_file_id = {
+ "model": "openai/gpt-5-mini",
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "file",
+ "file": {
+ "file_id": "file-abc123",
+ "filename": "document.pdf",
+ }
+ }
+ ]
+ }
+ ]
+ }
+
+ result = _convert_chat_request_to_responses_request(request_with_file_id)
+
+ content = result["input"][0]["content"]
+ assert len(content) == 1
+ assert content[0]["type"] == "input_file"
+ assert content[0]["file_id"] == "file-abc123"
+ assert content[0]["filename"] == "document.pdf"
+
+ # Test with all file fields
+ request_with_all_fields = {
+ "model": "openai/gpt-5-mini",
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "file",
+ "file": {
+ "file_data": "data:application/pdf;base64,JVBERi0xLjQ=",
+ "file_id": "file-xyz789",
+ "filename": "report.pdf",
+ }
+ }
+ ]
+ }
+ ]
+ }
+
+ result = _convert_chat_request_to_responses_request(request_with_all_fields)
+
+ content = result["input"][0]["content"]
+ assert content[0]["type"] == "input_file"
+ assert content[0]["file_data"] == "data:application/pdf;base64,JVBERi0xLjQ="
+ assert content[0]["file_id"] == "file-xyz789"
+ assert content[0]["filename"] == "report.pdf"
+
+
def test_responses_api_with_image_input():
api_response = make_response(
output_blocks=[
diff --git a/tests/signatures/test_adapter_file.py b/tests/signatures/test_adapter_file.py
new file mode 100644
index 000000000..e44ab900e
--- /dev/null
+++ b/tests/signatures/test_adapter_file.py
@@ -0,0 +1,274 @@
+import os
+import tempfile
+
+import pydantic
+import pytest
+
+import dspy
+from dspy.adapters.types.file import encode_file_to_dict
+from dspy.utils.dummies import DummyLM
+
+
+def sample_text_file():
+ with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as tmp_file:
+ tmp_file.write("This is a test file.")
+ tmp_file_path = tmp_file.name
+ yield tmp_file_path
+ try:
+ os.unlink(tmp_file_path)
+ except Exception:
+ pass
+
+
+def count_messages_with_file_pattern(messages):
+ pattern = {"type": "file", "file": lambda x: isinstance(x, dict)}
+
+ def check_pattern(obj, pattern):
+ if isinstance(pattern, dict):
+ if not isinstance(obj, dict):
+ return False
+ return all(k in obj and check_pattern(obj[k], v) for k, v in pattern.items())
+ if callable(pattern):
+ return pattern(obj)
+ return obj == pattern
+
+ def count_patterns(obj, pattern):
+ count = 0
+ if check_pattern(obj, pattern):
+ count += 1
+ if isinstance(obj, dict):
+ count += sum(count_patterns(v, pattern) for v in obj.values())
+ if isinstance(obj, list | tuple):
+ count += sum(count_patterns(v, pattern) for v in obj)
+ return count
+
+ return count_patterns(messages, pattern)
+
+
+def setup_predictor(signature, expected_output):
+ lm = DummyLM([expected_output])
+ dspy.settings.configure(lm=lm)
+ return dspy.Predict(signature), lm
+
+
+def test_file_from_local_path(sample_text_file):
+ file_obj = dspy.File.from_path(sample_text_file)
+ assert file_obj.file_data is not None
+ assert file_obj.file_data.startswith("data:text/plain;base64,")
+ assert file_obj.filename == os.path.basename(sample_text_file)
+
+
+def test_file_from_path_method(sample_text_file):
+ file_obj = dspy.File.from_path(sample_text_file)
+ assert file_obj.file_data is not None
+ assert file_obj.file_data.startswith("data:text/plain;base64,")
+ assert file_obj.filename == os.path.basename(sample_text_file)
+
+
+def test_file_from_path_with_custom_filename(sample_text_file):
+ file_obj = dspy.File.from_path(sample_text_file, filename="custom.txt")
+ assert file_obj.file_data is not None
+ assert file_obj.file_data.startswith("data:text/plain;base64,")
+ assert file_obj.filename == "custom.txt"
+
+
+def test_file_from_bytes():
+ file_bytes = b"Test file content"
+ file_obj = dspy.File.from_bytes(file_bytes)
+ assert file_obj.file_data is not None
+ assert file_obj.file_data.startswith("data:application/octet-stream;base64,")
+ assert file_obj.filename is None
+
+
+def test_file_from_bytes_with_filename():
+ file_bytes = b"Test file content"
+ file_obj = dspy.File.from_bytes(file_bytes, filename="test.txt")
+ assert file_obj.file_data is not None
+ assert file_obj.file_data.startswith("data:application/octet-stream;base64,")
+ assert file_obj.filename == "test.txt"
+
+
+def test_file_from_file_id():
+ file_obj = dspy.File.from_file_id("file-abc123")
+ assert file_obj.file_id == "file-abc123"
+ assert file_obj.file_data is None
+
+
+def test_file_from_file_id_with_filename():
+ file_obj = dspy.File.from_file_id("file-abc123", filename="document.pdf")
+ assert file_obj.file_id == "file-abc123"
+ assert file_obj.filename == "document.pdf"
+
+
+def test_file_from_dict_with_file_data():
+ file_obj = dspy.File(file_data="data:text/plain;base64,dGVzdA==", filename="test.txt")
+ assert file_obj.file_data == "data:text/plain;base64,dGVzdA=="
+ assert file_obj.filename == "test.txt"
+
+
+def test_file_from_dict_with_file_id():
+ file_obj = dspy.File(file_id="file-xyz789")
+ assert file_obj.file_id == "file-xyz789"
+
+
+def test_file_format_with_file_data():
+ file_obj = dspy.File.from_bytes(b"test", filename="test.txt")
+ formatted = file_obj.format()
+ assert isinstance(formatted, list)
+ assert len(formatted) == 1
+ assert formatted[0]["type"] == "file"
+ assert "file" in formatted[0]
+ assert "file_data" in formatted[0]["file"]
+ assert "filename" in formatted[0]["file"]
+
+
+def test_file_format_with_file_id():
+ file_obj = dspy.File.from_file_id("file-123")
+ formatted = file_obj.format()
+ assert formatted[0]["type"] == "file"
+ assert formatted[0]["file"]["file_id"] == "file-123"
+
+
+def test_file_repr_with_file_data():
+ file_obj = dspy.File.from_bytes(b"Test content", filename="test.txt")
+ repr_str = repr(file_obj)
+ assert "DATA_URI" in repr_str
+ assert "application/octet-stream" in repr_str
+ assert "filename='test.txt'" in repr_str
+
+
+def test_file_repr_with_file_id():
+ file_obj = dspy.File.from_file_id("file-abc", filename="doc.pdf")
+ repr_str = repr(file_obj)
+ assert "file_id='file-abc'" in repr_str
+ assert "filename='doc.pdf'" in repr_str
+
+
+def test_file_str():
+ file_obj = dspy.File.from_bytes(b"test")
+ str_repr = str(file_obj)
+ assert "<<CUSTOM-TYPE-START-IDENTIFIER>>" in str_repr
+ assert "<<CUSTOM-TYPE-END-IDENTIFIER>>" in str_repr
+
+
+def test_encode_file_to_dict_from_path(sample_text_file):
+ result = encode_file_to_dict(sample_text_file)
+ assert "file_data" in result
+ assert result["file_data"].startswith("data:text/plain;base64,")
+ assert "filename" in result
+
+
+def test_encode_file_to_dict_from_bytes():
+ result = encode_file_to_dict(b"test content")
+ assert "file_data" in result
+ assert result["file_data"].startswith("data:application/octet-stream;base64,")
+
+
+def test_invalid_file_string():
+ with pytest.raises(ValueError, match="Unrecognized"):
+ encode_file_to_dict("https://this_is_not_a_file_path")
+
+
+def test_invalid_dict():
+ with pytest.raises(ValueError, match="must contain at least one"):
+ dspy.File(invalid="dict")
+
+
+def test_file_in_signature(sample_text_file):
+ signature = "document: dspy.File -> summary: str"
+ expected = {"summary": "This is a summary"}
+ predictor, lm = setup_predictor(signature, expected)
+
+ file_obj = dspy.File.from_path(sample_text_file)
+ result = predictor(document=file_obj)
+
+ assert result.summary == "This is a summary"
+ assert count_messages_with_file_pattern(lm.history[-1]["messages"]) == 1
+
+
+def test_file_list_in_signature(sample_text_file):
+ class FileListSignature(dspy.Signature):
+ documents: list[dspy.File] = dspy.InputField()
+ summary: str = dspy.OutputField()
+
+ expected = {"summary": "Multiple files"}
+ predictor, lm = setup_predictor(FileListSignature, expected)
+
+ files = [
+ dspy.File.from_path(sample_text_file),
+ dspy.File.from_file_id("file-123"),
+ ]
+ result = predictor(documents=files)
+
+ assert result.summary == "Multiple files"
+ assert count_messages_with_file_pattern(lm.history[-1]["messages"]) == 2
+
+
+def test_optional_file_field():
+ class OptionalFileSignature(dspy.Signature):
+ document: dspy.File | None = dspy.InputField()
+ output: str = dspy.OutputField()
+
+ predictor, lm = setup_predictor(OptionalFileSignature, {"output": "Hello"})
+ result = predictor(document=None)
+ assert result.output == "Hello"
+ assert count_messages_with_file_pattern(lm.history[-1]["messages"]) == 0
+
+
+def test_save_load_file_signature(sample_text_file):
+ signature = "document: dspy.File -> summary: str"
+ file_obj = dspy.File.from_path(sample_text_file)
+ examples = [dspy.Example(document=file_obj, summary="Test summary")]
+
+ predictor, lm = setup_predictor(signature, {"summary": "A summary"})
+ optimizer = dspy.teleprompt.LabeledFewShot(k=1)
+ compiled_predictor = optimizer.compile(student=predictor, trainset=examples, sample=False)
+
+ with tempfile.NamedTemporaryFile(mode="w+", delete=True, suffix=".json") as temp_file:
+ compiled_predictor.save(temp_file.name)
+ loaded_predictor = dspy.Predict(signature)
+ loaded_predictor.load(temp_file.name)
+
+ loaded_predictor(document=dspy.File.from_file_id("file-test"))
+ assert count_messages_with_file_pattern(lm.history[-1]["messages"]) == 2
+
+
+def test_file_frozen():
+ file_obj = dspy.File.from_bytes(b"test")
+ with pytest.raises((TypeError, ValueError, pydantic.ValidationError)):
+ file_obj.file_data = "new_data"
+
+
+def test_file_with_all_fields():
+ file_data_uri = "data:text/plain;base64,dGVzdA=="
+ file_obj = dspy.File(file_data=file_data_uri, file_id="file-123", filename="test.txt")
+ assert file_obj.file_data == file_data_uri
+ assert file_obj.file_id == "file-123"
+ assert file_obj.filename == "test.txt"
+
+ formatted = file_obj.format()
+ assert formatted[0]["file"]["file_data"] == file_data_uri
+ assert formatted[0]["file"]["file_id"] == "file-123"
+ assert formatted[0]["file"]["filename"] == "test.txt"
+
+
+def test_file_path_not_found():
+ with pytest.raises(ValueError, match="File not found"):
+ dspy.File.from_path("/nonexistent/path/file.txt")
+
+
+def test_file_custom_mime_type(sample_text_file):
+ file_obj = dspy.File.from_path(sample_text_file, mime_type="text/custom")
+ assert file_obj.file_data.startswith("data:text/custom;base64,")
+
+
+def test_file_from_bytes_custom_mime():
+ file_obj = dspy.File.from_bytes(b"audio data", mime_type="audio/mp3")
+ assert file_obj.file_data.startswith("data:audio/mp3;base64,")
+
+
+def test_file_data_uri_in_format():
+ file_obj = dspy.File.from_bytes(b"test", filename="test.txt", mime_type="text/plain")
+ formatted = file_obj.format()
+ assert "data:text/plain;base64," in formatted[0]["file"]["file_data"]