Skip to content

The agent engineering platform.

core[patch]: extract input variables for `path` and `detail` keys in order to format an `ImagePromptTemplate` (#22613)

Théo Deschamps39b19cf76401
diff --git a/libs/core/langchain_core/prompts/chat.py b/libs/core/langchain_core/prompts/chat.py
index f0937f1a3c..9d27aedce8 100644
--- a/libs/core/langchain_core/prompts/chat.py
+++ b/libs/core/langchain_core/prompts/chat.py
@@ -473,6 +473,7 @@ class _StringImageMessagePromptTemplate(BaseMessagePromptTemplate):
)
elif isinstance(tmpl, dict) and "image_url" in tmpl:
img_template = cast(_ImageTemplateParam, tmpl)["image_url"]
+ input_variables = []
if isinstance(img_template, str):
vars = get_template_variables(img_template, "f-string")
if vars:
@@ -483,20 +484,19 @@ class _StringImageMessagePromptTemplate(BaseMessagePromptTemplate):
f"\nFrom: {tmpl}"
)
input_variables = [vars[0]]
- else:
- input_variables = None
img_template = {"url": img_template}
img_template_obj = ImagePromptTemplate(
input_variables=input_variables, template=img_template
)
elif isinstance(img_template, dict):
img_template = dict(img_template)
- if "url" in img_template:
- input_variables = get_template_variables(
- img_template["url"], "f-string"
- )
- else:
- input_variables = None
+ for key in ["url", "path", "detail"]:
+ if key in img_template:
+ input_variables.extend(
+ get_template_variables(
+ img_template[key], "f-string"
+ )
+ )
img_template_obj = ImagePromptTemplate(
input_variables=input_variables, template=img_template
)
diff --git a/libs/core/tests/unit_tests/prompts/test_chat.py b/libs/core/tests/unit_tests/prompts/test_chat.py
index 5811480f04..86f1cc1954 100644
--- a/libs/core/tests/unit_tests/prompts/test_chat.py
+++ b/libs/core/tests/unit_tests/prompts/test_chat.py
@@ -1,3 +1,5 @@
+import base64
+import tempfile
from pathlib import Path
from typing import Any, List, Union
@@ -559,6 +561,7 @@ async def test_chat_tmpl_from_messages_multipart_text_with_template() -> None:
async def test_chat_tmpl_from_messages_multipart_image() -> None:
+ """Test multipart image URL formatting."""
base64_image = "iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAA"
other_base64_image = "iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAA"
template = ChatPromptTemplate.from_messages(
@@ -641,6 +644,65 @@ async def test_chat_tmpl_from_messages_multipart_image() -> None:
assert messages == expected
+async def test_chat_tmpl_from_messages_multipart_formatting_with_path() -> None:
+ """Verify that we can pass `path` for an image as a variable."""
+ in_mem = "base64mem"
+ in_file_data = "base64file01"
+
+ with tempfile.NamedTemporaryFile(delete=True, suffix=".jpg") as temp_file:
+ temp_file.write(base64.b64decode(in_file_data))
+ temp_file.flush()
+
+ template = ChatPromptTemplate.from_messages(
+ [
+ ("system", "You are an AI assistant named {name}."),
+ (
+ "human",
+ [
+ {"type": "text", "text": "What's in this image?"},
+ {
+ "type": "image_url",
+ "image_url": "data:image/jpeg;base64,{in_mem}",
+ },
+ {
+ "type": "image_url",
+ "image_url": {"path": "{file_path}"},
+ },
+ ],
+ ),
+ ]
+ )
+ expected = [
+ SystemMessage(content="You are an AI assistant named R2D2."),
+ HumanMessage(
+ content=[
+ {"type": "text", "text": "What's in this image?"},
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/jpeg;base64,{in_mem}"},
+ },
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/jpeg;base64,{in_file_data}"},
+ },
+ ]
+ ),
+ ]
+ messages = template.format_messages(
+ name="R2D2",
+ in_mem=in_mem,
+ file_path=temp_file.name,
+ )
+ assert messages == expected
+
+ messages = await template.aformat_messages(
+ name="R2D2",
+ in_mem=in_mem,
+ file_path=temp_file.name,
+ )
+ assert messages == expected
+
+
def test_messages_placeholder() -> None:
prompt = MessagesPlaceholder("history")
with pytest.raises(KeyError):