DSPy: The framework for programming—not prompting—language models
Python
feat(File): add File type for handling file data (#9014)
TomuHiratac542bb64a7bb
diff --git a/dspy/__init__.py b/dspy/__init__.pyindex ea4c75a86..01aad3675 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -6,7 +6,7 @@ from dspy.teleprompt import *from dspy.evaluate import Evaluate # isort: skipfrom dspy.clients import * # isort: skip-from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, XMLAdapter, TwoStepAdapter, Image, Audio, History, Type, Tool, ToolCalls, Code # isort: skip+from dspy.adapters import Adapter, ChatAdapter, JSONAdapter, XMLAdapter, TwoStepAdapter, Image, Audio, File, History, Type, Tool, ToolCalls, Code # isort: skipfrom dspy.utils.logging_utils import configure_dspy_loggers, disable_logging, enable_loggingfrom dspy.utils.asyncify import asyncifyfrom dspy.utils.syncify import syncifydiff --git a/dspy/adapters/__init__.py b/dspy/adapters/__init__.pyindex 1dea6da47..fded25398 100644--- a/dspy/adapters/__init__.py+++ b/dspy/adapters/__init__.py@@ -2,7 +2,7 @@ from dspy.adapters.base import Adapterfrom dspy.adapters.chat_adapter import ChatAdapterfrom dspy.adapters.json_adapter import JSONAdapterfrom dspy.adapters.two_step_adapter import TwoStepAdapter-from dspy.adapters.types import Audio, Code, History, Image, Tool, ToolCalls, Type+from dspy.adapters.types import Audio, Code, File, History, Image, Tool, ToolCalls, Typefrom dspy.adapters.xml_adapter import XMLAdapter__all__ = [@@ -12,6 +12,7 @@ __all__ = ["History","Image","Audio",+ "File","Code","JSONAdapter","XMLAdapter",diff --git a/dspy/adapters/types/__init__.py b/dspy/adapters/types/__init__.pyindex 11b9faee1..eb1481c86 100644--- a/dspy/adapters/types/__init__.py+++ b/dspy/adapters/types/__init__.py@@ -1,8 +1,9 @@from dspy.adapters.types.audio import Audiofrom dspy.adapters.types.base_type import Typefrom dspy.adapters.types.code import Code+from dspy.adapters.types.file import Filefrom dspy.adapters.types.history import Historyfrom dspy.adapters.types.image import Imagefrom dspy.adapters.types.tool import Tool, ToolCalls-__all__ = ["History", "Image", "Audio", "Type", "Tool", "ToolCalls", "Code"]+__all__ = ["History", "Image", "Audio", "File", "Type", "Tool", "ToolCalls", "Code"]diff --git a/dspy/adapters/types/file.py b/dspy/adapters/types/file.pynew file mode 100644index 000000000..34bc2a011--- /dev/null+++ b/dspy/adapters/types/file.py@@ -0,0 +1,178 @@+import base64+import mimetypes+import os+from typing import Any++import pydantic++from dspy.adapters.types.base_type import Type+++class File(Type):+ """A file input type for DSPy.+ See https://platform.openai.com/docs/api-reference/chat/create#chat_create-messages-user_message-content-array_of_content_parts-file_content_part-file for specification.++ The file_data field should be a data URI with the format:+ data:<mime_type>;base64,<base64_encoded_data>++ Example:+ ```python+ import dspy++ class QA(dspy.Signature):+ file: dspy.File = dspy.InputField()+ summary = dspy.OutputField()+ program = dspy.Predict(QA)+ result = program(file=dspy.File.from_path("./research.pdf"))+ print(result.summary)+ ```+ """++ file_data: str | None = None+ file_id: str | None = None+ filename: str | None = None++ model_config = pydantic.ConfigDict(+ frozen=True,+ str_strip_whitespace=True,+ validate_assignment=True,+ extra="forbid",+ )++ @pydantic.model_validator(mode="before")+ @classmethod+ def validate_input(cls, values: Any) -> Any:+ if isinstance(values, cls):+ return {+ "file_data": values.file_data,+ "file_id": values.file_id,+ "filename": values.filename,+ }++ if isinstance(values, dict):+ if "file_data" in values or "file_id" in values or "filename" in values:+ return values+ raise ValueError("Value of `dspy.File` must contain at least one of: file_data, file_id, or filename")++ return encode_file_to_dict(values)++ def format(self) -> list[dict[str, Any]]:+ try:+ file_dict = {}+ if self.file_data:+ file_dict["file_data"] = self.file_data+ if self.file_id:+ file_dict["file_id"] = self.file_id+ if self.filename:+ file_dict["filename"] = self.filename++ return [{"type": "file", "file": file_dict}]+ except Exception as e:+ raise ValueError(f"Failed to format file for DSPy: {e}")++ def __str__(self):+ return self.serialize_model()++ def __repr__(self):+ parts = []+ if self.file_data is not None:+ if self.file_data.startswith("data:"):+ # file data has "data:text/plain;base64,..." format+ mime_type = self.file_data.split(";")[0].split(":")[1]+ len_data = len(self.file_data.split("base64,")[1]) if "base64," in self.file_data else len(self.file_data)+ parts.append(f"file_data=<DATA_URI({mime_type}, {len_data} chars)>")+ else:+ len_data = len(self.file_data)+ parts.append(f"file_data=<DATA({len_data} chars)>")+ if self.file_id is not None:+ parts.append(f"file_id='{self.file_id}'")+ if self.filename is not None:+ parts.append(f"filename='{self.filename}'")+ return f"File({', '.join(parts)})"++ @classmethod+ def from_path(cls, file_path: str, filename: str | None = None, mime_type: str | None = None) -> "File":+ """Create a File from a local file path.++ Args:+ file_path: Path to the file to read+ filename: Optional filename to use (defaults to basename of path)+ mime_type: Optional MIME type (defaults to auto-detection from file extension)+ """+ if not os.path.isfile(file_path):+ raise ValueError(f"File not found: {file_path}")++ with open(file_path, "rb") as f:+ file_bytes = f.read()++ if filename is None:+ filename = os.path.basename(file_path)++ if mime_type is None:+ mime_type, _ = mimetypes.guess_type(file_path)+ if mime_type is None:+ mime_type = "application/octet-stream"++ encoded_data = base64.b64encode(file_bytes).decode("utf-8")+ file_data = f"data:{mime_type};base64,{encoded_data}"++ return cls(file_data=file_data, filename=filename)++ @classmethod+ def from_bytes(+ cls, file_bytes: bytes, filename: str | None = None, mime_type: str = "application/octet-stream"+ ) -> "File":+ """Create a File from raw bytes.++ Args:+ file_bytes: Raw bytes of the file+ filename: Optional filename+ mime_type: MIME type (defaults to 'application/octet-stream')+ """+ encoded_data = base64.b64encode(file_bytes).decode("utf-8")+ file_data = f"data:{mime_type};base64,{encoded_data}"+ return cls(file_data=file_data, filename=filename)++ @classmethod+ def from_file_id(cls, file_id: str, filename: str | None = None) -> "File":+ """Create a File from an uploaded file ID."""+ return cls(file_id=file_id, filename=filename)+++def encode_file_to_dict(file_input: Any) -> dict:+ """+ Encode various file inputs to a dict with file_data, file_id, and/or filename.++ Args:+ file_input: Can be a file path (str), bytes, or File instance.++ Returns:+ dict: A dictionary with file_data, file_id, and/or filename keys.+ """+ if isinstance(file_input, File):+ result = {}+ if file_input.file_data is not None:+ result["file_data"] = file_input.file_data+ if file_input.file_id is not None:+ result["file_id"] = file_input.file_id+ if file_input.filename is not None:+ result["filename"] = file_input.filename+ return result++ elif isinstance(file_input, str):+ if os.path.isfile(file_input):+ file_obj = File.from_path(file_input)+ else:+ raise ValueError(f"Unrecognized file string: {file_input}; must be a valid file path")++ return {+ "file_data": file_obj.file_data,+ "filename": file_obj.filename,+ }++ elif isinstance(file_input, bytes):+ file_obj = File.from_bytes(file_input)+ return {"file_data": file_obj.file_data}++ else:+ raise ValueError(f"Unsupported file input type: {type(file_input)}")diff --git a/dspy/clients/lm.py b/dspy/clients/lm.pyindex f9fc648ca..81d968703 100644--- a/dspy/clients/lm.py+++ b/dspy/clients/lm.py@@ -476,6 +476,11 @@ async def alitellm_responses_completion(request: dict[str, Any], num_retries: indef _convert_chat_request_to_responses_request(request: dict[str, Any]):+ """+ Convert a chat request to a responses request+ See https://platform.openai.com/docs/api-reference/responses/create for the responses API specification.+ Also see https://platform.openai.com/docs/api-reference/chat/create for the chat API specification.+ """request = dict(request)if "messages" in request:content_blocks = []@@ -525,6 +530,14 @@ def _convert_content_item_to_responses_format(item: dict[str, Any]) -> dict[str,"type": "input_text","text": item.get("text", ""),}+ elif item.get("type") == "file":+ file = item.get("file", {})+ return {+ "type": "input_file",+ "file_data": file.get("file_data"),+ "filename": file.get("filename"),+ "file_id": file.get("file_id"),+ }# For other items, return as-isreturn itemdiff --git a/dspy/utils/inspect_history.py b/dspy/utils/inspect_history.pyindex 556913a74..07934157f 100644--- a/dspy/utils/inspect_history.py+++ b/dspy/utils/inspect_history.py@@ -46,6 +46,13 @@ def pretty_print_history(history, n: int = 1):len_audio = len(c["input_audio"]["data"])audio_str = f"<audio format='{audio_format}' base64-encoded, length={len_audio}>"print(_blue(audio_str.strip()))+ elif c["type"] == "file" or c["type"] == "input_file":+ file = c.get("file", c.get("input_file", {}))+ filename = file.get("filename", "")+ file_id = file.get("file_id", "")+ file_data = file.get("file_data", "")+ file_str = f"<file: name:{filename}, id:{file_id}, data_length:{len(file_data)}>"+ print(_blue(file_str.strip()))print("\n")if isinstance(outputs[0], dict):diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.pyindex 64e76ecd9..3a228286f 100644--- a/tests/clients/test_lm.py+++ b/tests/clients/test_lm.py@@ -716,6 +716,103 @@ def test_responses_api_converts_images_correctly():assert content[0]["image_url"] == "https://example.com/image.jpg"+def test_responses_api_converts_files_correctly():+ from dspy.clients.lm import _convert_chat_request_to_responses_request++ # Test with file data (base64 encoded)+ request_with_file = {+ "model": "openai/gpt-5-mini",+ "messages": [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "Analyze this file"},+ {+ "type": "file",+ "file": {+ "file_data": "data:text/plain;base64,SGVsbG8gV29ybGQ=",+ "filename": "test.txt",+ }+ }+ ]+ }+ ]+ }++ result = _convert_chat_request_to_responses_request(request_with_file)++ assert "input" in result+ assert len(result["input"]) == 1+ assert result["input"][0]["role"] == "user"++ content = result["input"][0]["content"]+ assert len(content) == 2++ # First item should be text converted to input_text format+ assert content[0]["type"] == "input_text"+ assert content[0]["text"] == "Analyze this file"++ # Second item should be converted to input_file format+ assert content[1]["type"] == "input_file"+ assert content[1]["file_data"] == "data:text/plain;base64,SGVsbG8gV29ybGQ="+ assert content[1]["filename"] == "test.txt"++ # Test with file_id+ request_with_file_id = {+ "model": "openai/gpt-5-mini",+ "messages": [+ {+ "role": "user",+ "content": [+ {+ "type": "file",+ "file": {+ "file_id": "file-abc123",+ "filename": "document.pdf",+ }+ }+ ]+ }+ ]+ }++ result = _convert_chat_request_to_responses_request(request_with_file_id)++ content = result["input"][0]["content"]+ assert len(content) == 1+ assert content[0]["type"] == "input_file"+ assert content[0]["file_id"] == "file-abc123"+ assert content[0]["filename"] == "document.pdf"++ # Test with all file fields+ request_with_all_fields = {+ "model": "openai/gpt-5-mini",+ "messages": [+ {+ "role": "user",+ "content": [+ {+ "type": "file",+ "file": {+ "file_data": "data:application/pdf;base64,JVBERi0xLjQ=",+ "file_id": "file-xyz789",+ "filename": "report.pdf",+ }+ }+ ]+ }+ ]+ }++ result = _convert_chat_request_to_responses_request(request_with_all_fields)++ content = result["input"][0]["content"]+ assert content[0]["type"] == "input_file"+ assert content[0]["file_data"] == "data:application/pdf;base64,JVBERi0xLjQ="+ assert content[0]["file_id"] == "file-xyz789"+ assert content[0]["filename"] == "report.pdf"++def test_responses_api_with_image_input():api_response = make_response(output_blocks=[diff --git a/tests/signatures/test_adapter_file.py b/tests/signatures/test_adapter_file.pynew file mode 100644index 000000000..e44ab900e--- /dev/null+++ b/tests/signatures/test_adapter_file.py@@ -0,0 +1,274 @@+import os+import tempfile++import pydantic+import pytest++import dspy+from dspy.adapters.types.file import encode_file_to_dict+from dspy.utils.dummies import DummyLM+++def sample_text_file():+ with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as tmp_file:+ tmp_file.write("This is a test file.")+ tmp_file_path = tmp_file.name+ yield tmp_file_path+ try:+ os.unlink(tmp_file_path)+ except Exception:+ pass+++def count_messages_with_file_pattern(messages):+ pattern = {"type": "file", "file": lambda x: isinstance(x, dict)}++ def check_pattern(obj, pattern):+ if isinstance(pattern, dict):+ if not isinstance(obj, dict):+ return False+ return all(k in obj and check_pattern(obj[k], v) for k, v in pattern.items())+ if callable(pattern):+ return pattern(obj)+ return obj == pattern++ def count_patterns(obj, pattern):+ count = 0+ if check_pattern(obj, pattern):+ count += 1+ if isinstance(obj, dict):+ count += sum(count_patterns(v, pattern) for v in obj.values())+ if isinstance(obj, list | tuple):+ count += sum(count_patterns(v, pattern) for v in obj)+ return count++ return count_patterns(messages, pattern)+++def setup_predictor(signature, expected_output):+ lm = DummyLM([expected_output])+ dspy.settings.configure(lm=lm)+ return dspy.Predict(signature), lm+++def test_file_from_local_path(sample_text_file):+ file_obj = dspy.File.from_path(sample_text_file)+ assert file_obj.file_data is not None+ assert file_obj.file_data.startswith("data:text/plain;base64,")+ assert file_obj.filename == os.path.basename(sample_text_file)+++def test_file_from_path_method(sample_text_file):+ file_obj = dspy.File.from_path(sample_text_file)+ assert file_obj.file_data is not None+ assert file_obj.file_data.startswith("data:text/plain;base64,")+ assert file_obj.filename == os.path.basename(sample_text_file)+++def test_file_from_path_with_custom_filename(sample_text_file):+ file_obj = dspy.File.from_path(sample_text_file, filename="custom.txt")+ assert file_obj.file_data is not None+ assert file_obj.file_data.startswith("data:text/plain;base64,")+ assert file_obj.filename == "custom.txt"+++def test_file_from_bytes():+ file_bytes = b"Test file content"+ file_obj = dspy.File.from_bytes(file_bytes)+ assert file_obj.file_data is not None+ assert file_obj.file_data.startswith("data:application/octet-stream;base64,")+ assert file_obj.filename is None+++def test_file_from_bytes_with_filename():+ file_bytes = b"Test file content"+ file_obj = dspy.File.from_bytes(file_bytes, filename="test.txt")+ assert file_obj.file_data is not None+ assert file_obj.file_data.startswith("data:application/octet-stream;base64,")+ assert file_obj.filename == "test.txt"+++def test_file_from_file_id():+ file_obj = dspy.File.from_file_id("file-abc123")+ assert file_obj.file_id == "file-abc123"+ assert file_obj.file_data is None+++def test_file_from_file_id_with_filename():+ file_obj = dspy.File.from_file_id("file-abc123", filename="document.pdf")+ assert file_obj.file_id == "file-abc123"+ assert file_obj.filename == "document.pdf"+++def test_file_from_dict_with_file_data():+ file_obj = dspy.File(file_data="data:text/plain;base64,dGVzdA==", filename="test.txt")+ assert file_obj.file_data == "data:text/plain;base64,dGVzdA=="+ assert file_obj.filename == "test.txt"+++def test_file_from_dict_with_file_id():+ file_obj = dspy.File(file_id="file-xyz789")+ assert file_obj.file_id == "file-xyz789"+++def test_file_format_with_file_data():+ file_obj = dspy.File.from_bytes(b"test", filename="test.txt")+ formatted = file_obj.format()+ assert isinstance(formatted, list)+ assert len(formatted) == 1+ assert formatted[0]["type"] == "file"+ assert "file" in formatted[0]+ assert "file_data" in formatted[0]["file"]+ assert "filename" in formatted[0]["file"]+++def test_file_format_with_file_id():+ file_obj = dspy.File.from_file_id("file-123")+ formatted = file_obj.format()+ assert formatted[0]["type"] == "file"+ assert formatted[0]["file"]["file_id"] == "file-123"+++def test_file_repr_with_file_data():+ file_obj = dspy.File.from_bytes(b"Test content", filename="test.txt")+ repr_str = repr(file_obj)+ assert "DATA_URI" in repr_str+ assert "application/octet-stream" in repr_str+ assert "filename='test.txt'" in repr_str+++def test_file_repr_with_file_id():+ file_obj = dspy.File.from_file_id("file-abc", filename="doc.pdf")+ repr_str = repr(file_obj)+ assert "file_id='file-abc'" in repr_str+ assert "filename='doc.pdf'" in repr_str+++def test_file_str():+ file_obj = dspy.File.from_bytes(b"test")+ str_repr = str(file_obj)+ assert "<<CUSTOM-TYPE-START-IDENTIFIER>>" in str_repr+ assert "<<CUSTOM-TYPE-END-IDENTIFIER>>" in str_repr+++def test_encode_file_to_dict_from_path(sample_text_file):+ result = encode_file_to_dict(sample_text_file)+ assert "file_data" in result+ assert result["file_data"].startswith("data:text/plain;base64,")+ assert "filename" in result+++def test_encode_file_to_dict_from_bytes():+ result = encode_file_to_dict(b"test content")+ assert "file_data" in result+ assert result["file_data"].startswith("data:application/octet-stream;base64,")+++def test_invalid_file_string():+ with pytest.raises(ValueError, match="Unrecognized"):+ encode_file_to_dict("https://this_is_not_a_file_path")+++def test_invalid_dict():+ with pytest.raises(ValueError, match="must contain at least one"):+ dspy.File(invalid="dict")+++def test_file_in_signature(sample_text_file):+ signature = "document: dspy.File -> summary: str"+ expected = {"summary": "This is a summary"}+ predictor, lm = setup_predictor(signature, expected)++ file_obj = dspy.File.from_path(sample_text_file)+ result = predictor(document=file_obj)++ assert result.summary == "This is a summary"+ assert count_messages_with_file_pattern(lm.history[-1]["messages"]) == 1+++def test_file_list_in_signature(sample_text_file):+ class FileListSignature(dspy.Signature):+ documents: list[dspy.File] = dspy.InputField()+ summary: str = dspy.OutputField()++ expected = {"summary": "Multiple files"}+ predictor, lm = setup_predictor(FileListSignature, expected)++ files = [+ dspy.File.from_path(sample_text_file),+ dspy.File.from_file_id("file-123"),+ ]+ result = predictor(documents=files)++ assert result.summary == "Multiple files"+ assert count_messages_with_file_pattern(lm.history[-1]["messages"]) == 2+++def test_optional_file_field():+ class OptionalFileSignature(dspy.Signature):+ document: dspy.File | None = dspy.InputField()+ output: str = dspy.OutputField()++ predictor, lm = setup_predictor(OptionalFileSignature, {"output": "Hello"})+ result = predictor(document=None)+ assert result.output == "Hello"+ assert count_messages_with_file_pattern(lm.history[-1]["messages"]) == 0+++def test_save_load_file_signature(sample_text_file):+ signature = "document: dspy.File -> summary: str"+ file_obj = dspy.File.from_path(sample_text_file)+ examples = [dspy.Example(document=file_obj, summary="Test summary")]++ predictor, lm = setup_predictor(signature, {"summary": "A summary"})+ optimizer = dspy.teleprompt.LabeledFewShot(k=1)+ compiled_predictor = optimizer.compile(student=predictor, trainset=examples, sample=False)++ with tempfile.NamedTemporaryFile(mode="w+", delete=True, suffix=".json") as temp_file:+ compiled_predictor.save(temp_file.name)+ loaded_predictor = dspy.Predict(signature)+ loaded_predictor.load(temp_file.name)++ loaded_predictor(document=dspy.File.from_file_id("file-test"))+ assert count_messages_with_file_pattern(lm.history[-1]["messages"]) == 2+++def test_file_frozen():+ file_obj = dspy.File.from_bytes(b"test")+ with pytest.raises((TypeError, ValueError, pydantic.ValidationError)):+ file_obj.file_data = "new_data"+++def test_file_with_all_fields():+ file_data_uri = "data:text/plain;base64,dGVzdA=="+ file_obj = dspy.File(file_data=file_data_uri, file_id="file-123", filename="test.txt")+ assert file_obj.file_data == file_data_uri+ assert file_obj.file_id == "file-123"+ assert file_obj.filename == "test.txt"++ formatted = file_obj.format()+ assert formatted[0]["file"]["file_data"] == file_data_uri+ assert formatted[0]["file"]["file_id"] == "file-123"+ assert formatted[0]["file"]["filename"] == "test.txt"+++def test_file_path_not_found():+ with pytest.raises(ValueError, match="File not found"):+ dspy.File.from_path("/nonexistent/path/file.txt")+++def test_file_custom_mime_type(sample_text_file):+ file_obj = dspy.File.from_path(sample_text_file, mime_type="text/custom")+ assert file_obj.file_data.startswith("data:text/custom;base64,")+++def test_file_from_bytes_custom_mime():+ file_obj = dspy.File.from_bytes(b"audio data", mime_type="audio/mp3")+ assert file_obj.file_data.startswith("data:audio/mp3;base64,")+++def test_file_data_uri_in_format():+ file_obj = dspy.File.from_bytes(b"test", filename="test.txt", mime_type="text/plain")+ formatted = file_obj.format()+ assert "data:text/plain;base64," in formatted[0]["file"]["file_data"]