Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Refactor image content to use a shared `ImageReference` type (#45543)

Krish Chainani5a66d460d3cb
diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json
index d2a9de926a..d05de8638d 100644
--- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json
+++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json
@@ -660,6 +660,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -671,9 +685,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -683,7 +694,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageContentItem",
@@ -1420,6 +1430,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -1431,9 +1455,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -1443,7 +1464,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json
index 83aa38fdcf..6d7bc110d5 100644
--- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json
+++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json
@@ -1759,6 +1759,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -1770,9 +1784,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -1782,7 +1793,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json
index ac76ffbb2d..813df4d7f3 100644
--- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json
+++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json
@@ -10472,6 +10472,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -10483,9 +10497,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -10495,7 +10506,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageContentItem",
@@ -12333,6 +12343,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -12344,9 +12368,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -12356,7 +12377,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json
index 7a95a06a2a..0e4c577b0c 100644
--- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json
+++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json
@@ -6352,6 +6352,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -6363,9 +6377,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -6375,7 +6386,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageContentItem",
@@ -8213,6 +8223,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -8224,9 +8248,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -8236,7 +8257,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json
index f3caf84e96..dc2d0dc619 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json
@@ -359,6 +359,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -370,9 +384,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -382,7 +393,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json
index 8432eb5437..c153e128b3 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json
@@ -359,6 +359,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -370,9 +384,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -382,7 +393,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json
index c6d8c66fe1..c05b788d8a 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json
@@ -80,6 +80,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -91,9 +105,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -103,7 +114,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageContentItem",
@@ -188,6 +198,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -199,9 +223,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -211,7 +232,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json
index 7da4ff34d8..cddc22f59f 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json
@@ -499,6 +499,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -510,9 +524,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -522,7 +533,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json
index 8a7fa76e53..a10d95cf00 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json
@@ -580,6 +580,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -591,9 +605,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -603,7 +614,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json
index d11f47942a..f467944e67 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json
@@ -359,6 +359,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -370,9 +384,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -382,7 +393,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json
index 32101efd89..537af065c3 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json
@@ -502,6 +502,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -513,9 +527,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -525,7 +536,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json
index f411af6d60..c76c1d2491 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json
@@ -502,6 +502,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -513,9 +527,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -525,7 +536,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json
index a639e719b8..510c26c2ee 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json
@@ -502,6 +502,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -513,9 +527,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -525,7 +536,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json
index 5c35287d0e..753f062836 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json
@@ -142,6 +142,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -153,9 +167,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -165,7 +176,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageContentItem",
@@ -250,6 +260,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -261,9 +285,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -273,7 +294,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json
index 2019868698..3a6fb8bdaa 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json
@@ -596,6 +596,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -607,9 +621,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -619,7 +630,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json
index 8dd0da9b6a..8d8a69f51d 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json
@@ -502,6 +502,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -513,9 +527,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -525,7 +536,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json
index 3f193b0de8..da04bf0a27 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json
@@ -580,6 +580,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -591,9 +605,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -603,7 +614,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json
index e7eb265dcb..084cc6522d 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json
@@ -502,6 +502,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -513,9 +527,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -525,7 +536,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json
index c8d1dd0354..3e06a271fa 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json
@@ -499,6 +499,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -510,9 +524,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -522,7 +533,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json
index 7c0f053e20..d396062af1 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json
@@ -502,6 +502,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -513,9 +527,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -525,7 +536,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json
index 7db8c18c82..9ca8c3b087 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json
@@ -499,6 +499,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -510,9 +524,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -522,7 +533,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json
index 9f8637ea17..68f4b3dff9 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json
@@ -166,6 +166,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -177,9 +191,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -189,7 +200,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json
index 9552369480..f2d2251e03 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json
@@ -499,6 +499,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -510,9 +524,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -522,7 +533,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json
index 57052afd9f..bdf800c52a 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json
@@ -499,6 +499,20 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "image_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "image_url"
+ ],
+ "title": "ImageUrlFunctionCallOutputContentItem",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -510,9 +524,6 @@
}
]
},
- "image_url": {
- "type": "string"
- },
"type": {
"enum": [
"input_image"
@@ -522,7 +533,6 @@
}
},
"required": [
- "image_url",
"type"
],
"title": "InputImageFunctionCallOutputContentItem",
diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst
index b4f9045ce0..79d6a3dfb1 100644
Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst differ
diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst
index 8222eefe7f..14bdda9860 100644
Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst differ
diff --git a/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts b/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts
index 9e53b5fc25..dd5b0a1929 100644
--- a/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts
+++ b/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts
@@ -3,4 +3,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { ImageDetail } from "./ImageDetail";
-export type ContentItem = { "type": "input_text", text: string, } | { "type": "input_image", image_url: string, detail?: ImageDetail, } | { "type": "input_audio", audio_url: string, } | { "type": "output_text", text: string, };
+export type ContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "output_text", text: string, };
diff --git a/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts b/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts
index 6c2ab2afa1..8ebefc596e 100644
--- a/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts
+++ b/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts
@@ -7,4 +7,4 @@ import type { ImageDetail } from "./ImageDetail";
* Responses API compatible content items that can be returned by a tool call.
* This is a subset of ContentItem with the types we support as function call outputs.
*/
-export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", image_url: string, detail?: ImageDetail, } | { "type": "input_audio", audio_url: string, } | { "type": "encrypted_content", encrypted_content: string, };
+export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "encrypted_content", encrypted_content: string, };
diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs
index 95e1d62f5c..a051c867db 100644
--- a/codex-rs/app-server/src/request_processors/turn_processor.rs
+++ b/codex-rs/app-server/src/request_processors/turn_processor.rs
@@ -8,6 +8,7 @@ use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
+use codex_protocol::models::ImageReference;
use codex_protocol::protocol::AdditionalContextEntry as CoreAdditionalContextEntry;
use codex_protocol::protocol::AdditionalContextKind as CoreAdditionalContextKind;
use codex_protocol::protocol::TurnSettingsUpdate;
@@ -36,7 +37,7 @@ fn validate_response_item_image_urls(items: &[ResponseItem]) -> Result<(), JSONR
ResponseItem::Message { content, .. } => content.iter().any(|item| {
matches!(
item,
- ContentItem::InputImage { image_url, .. } if is_remote_image_url(image_url)
+ ContentItem::InputImage { image: ImageReference::Inline { image_url }, .. } if is_remote_image_url(image_url)
)
}),
ResponseItem::FunctionCallOutput { output, .. }
@@ -45,7 +46,7 @@ fn validate_response_item_image_urls(items: &[ResponseItem]) -> Result<(), JSONR
content.iter().any(|item| {
matches!(
item,
- FunctionCallOutputContentItem::InputImage { image_url, .. }
+ FunctionCallOutputContentItem::InputImage { image: ImageReference::Inline { image_url }, .. }
if is_remote_image_url(image_url)
)
})
diff --git a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs
index 38cad9c430..38f34330ba 100644
--- a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs
+++ b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs
@@ -30,6 +30,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
+use codex_protocol::models::ImageReference;
use codex_protocol::openai_models::InputModality;
use core_test_support::load_default_config_for_test;
use core_test_support::responses;
@@ -715,7 +716,9 @@ async fn dynamic_tool_call_round_trip_handles_content_items() -> Result<()> {
text: "dynamic-ok".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: TINY_PNG_DATA_URL.to_string(),
+ image: ImageReference::Inline {
+ image_url: TINY_PNG_DATA_URL.to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
diff --git a/codex-rs/app-server/tests/suite/v2/request_validation.rs b/codex-rs/app-server/tests/suite/v2/request_validation.rs
index 1b5abe2f17..042236ed8e 100644
--- a/codex-rs/app-server/tests/suite/v2/request_validation.rs
+++ b/codex-rs/app-server/tests/suite/v2/request_validation.rs
@@ -14,6 +14,7 @@ use codex_app_server_protocol::ThreadStartResponse;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use pretty_assertions::assert_eq;
use serde_json::json;
@@ -173,7 +174,9 @@ async fn request_handlers_reject_remote_image_urls() -> Result<()> {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
- image_url: "https://example.com/tool.png".to_string(),
+ image: ImageReference::Inline {
+ image_url: "https://example.com/tool.png".to_string(),
+ },
detail: Some(ImageDetail::High),
},
]),
diff --git a/codex-rs/codex-api/src/endpoint/search.rs b/codex-rs/codex-api/src/endpoint/search.rs
index 131a335e26..6b2450e649 100644
--- a/codex-rs/codex-api/src/endpoint/search.rs
+++ b/codex-rs/codex-api/src/endpoint/search.rs
@@ -72,6 +72,7 @@ mod tests {
use codex_client::TransportError;
use codex_protocol::ResponseItemId;
use codex_protocol::models::ContentItem;
+ use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use http::StatusCode;
use pretty_assertions::assert_eq;
@@ -164,7 +165,9 @@ mod tests {
text: "find this".to_string(),
},
ContentItem::InputImage {
- image_url: "https://example.com/image.png".to_string(),
+ image: ImageReference::Inline {
+ image_url: "https://example.com/image.png".to_string(),
+ },
detail: None,
},
],
diff --git a/codex-rs/core/src/client_common_tests.rs b/codex-rs/core/src/client_common_tests.rs
index e8b030e6a1..1cda4ef13d 100644
--- a/codex-rs/core/src/client_common_tests.rs
+++ b/codex-rs/core/src/client_common_tests.rs
@@ -6,6 +6,7 @@ use codex_models_manager::model_info::model_info_from_slug;
use codex_protocol::config_types::ServiceTier;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use pretty_assertions::assert_eq;
use serde_json::value::RawValue;
use std::sync::Arc;
@@ -27,7 +28,9 @@ fn prompt_with_image_outputs(detail: Option<ImageDetail>) -> Prompt {
text: "Describe this image.".to_string(),
},
ContentItem::InputImage {
- image_url: "https://example.com/image.png".to_string(),
+ image: ImageReference::Inline {
+ image_url: "https://example.com/image.png".to_string(),
+ },
detail,
},
],
@@ -41,7 +44,9 @@ fn prompt_with_image_outputs(detail: Option<ImageDetail>) -> Prompt {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,function".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,function".to_string(),
+ },
detail,
},
]),
@@ -53,7 +58,9 @@ fn prompt_with_image_outputs(detail: Option<ImageDetail>) -> Prompt {
name: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,custom".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,custom".to_string(),
+ },
detail,
},
]),
diff --git a/codex-rs/core/src/compact_remote_v2.rs b/codex-rs/core/src/compact_remote_v2.rs
index 7a5d3ad416..b367ba4bea 100644
--- a/codex-rs/core/src/compact_remote_v2.rs
+++ b/codex-rs/core/src/compact_remote_v2.rs
@@ -45,6 +45,8 @@ use codex_protocol::items::ContextCompactionItem;
use codex_protocol::items::TurnItem;
use codex_protocol::models::AgentMessageInputContent;
use codex_protocol::models::ContentItem;
+#[cfg(test)]
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::TokenUsage;
@@ -981,11 +983,15 @@ mod tests {
text: "user".to_string(),
},
ContentItem::InputImage {
- image_url: "data:image/png;base64,abc".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,abc".to_string(),
+ },
detail: None,
},
ContentItem::InputImage {
- image_url: "data:image/png;base64,def".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,def".to_string(),
+ },
detail: None,
},
],
@@ -1047,7 +1053,9 @@ mod tests {
text: "abcdef".to_string(),
},
ContentItem::InputImage {
- image_url: "data:image/png;base64,abc".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,abc".to_string(),
+ },
detail: None,
},
ContentItem::OutputText {
@@ -1057,7 +1065,9 @@ mod tests {
text: "discarded after the text budget is exhausted".to_string(),
},
ContentItem::InputImage {
- image_url: "data:image/png;base64,def".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,def".to_string(),
+ },
detail: None,
},
],
@@ -1089,14 +1099,18 @@ mod tests {
text: "abcdef".to_string(),
},
ContentItem::InputImage {
- image_url: "data:image/png;base64,abc".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,abc".to_string()
+ },
detail: None,
},
ContentItem::OutputText {
text: "uv…1 tokens truncated…yz".to_string(),
},
ContentItem::InputImage {
- image_url: "data:image/png;base64,def".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,def".to_string()
+ },
detail: None,
},
],
@@ -1123,7 +1137,9 @@ mod tests {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage {
- image_url: "data:image/png;base64,abc".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,abc".to_string(),
+ },
detail: None,
}],
phase: None,
@@ -1147,7 +1163,9 @@ mod tests {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage {
- image_url: "data:image/png;base64,abc".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,abc".to_string(),
+ },
detail: None,
}],
phase: None,
diff --git a/codex-rs/core/src/compact_remote_v2_image_budget_tests.rs b/codex-rs/core/src/compact_remote_v2_image_budget_tests.rs
index ffc3a6f85a..7c7d94ac8b 100644
--- a/codex-rs/core/src/compact_remote_v2_image_budget_tests.rs
+++ b/codex-rs/core/src/compact_remote_v2_image_budget_tests.rs
@@ -1,5 +1,6 @@
use super::*;
use codex_protocol::models::ContentItemKind;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::InternalChatMessageMetadataPassthrough;
use codex_protocol::models::image_close_tag_text;
use codex_protocol::models::local_image_open_tag_text_with_path;
@@ -17,7 +18,9 @@ fn message(content: Vec<ContentItem>) -> ResponseItem {
fn image() -> ContentItem {
ContentItem::InputImage {
- image_url: "data:image/png;base64,abc".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,abc".to_string(),
+ },
detail: None,
}
}
diff --git a/codex-rs/core/src/compact_remote_v2_images.rs b/codex-rs/core/src/compact_remote_v2_images.rs
index d2fecb03a2..0575e07f01 100644
--- a/codex-rs/core/src/compact_remote_v2_images.rs
+++ b/codex-rs/core/src/compact_remote_v2_images.rs
@@ -4,6 +4,7 @@ use codex_context_fragments::set_annotated_content;
use codex_context_fragments::to_annotated_content;
use codex_history::ResponseItemEnvelope;
use codex_protocol::models::ContentItem;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::is_image_close_tag_text;
use codex_protocol::models::is_image_open_tag_text;
use codex_protocol::models::is_local_image_open_tag_text;
@@ -17,9 +18,12 @@ pub(super) fn content_item_token_count(item: &ContentItem) -> usize {
ContentItem::InputText { text } | ContentItem::OutputText { text } => {
approx_token_count(text)
}
- ContentItem::InputImage { image_url, detail } => usize::try_from(
- approx_tokens_from_byte_count_i64(estimate_image_bytes(image_url, *detail)),
- )
+ ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail,
+ } => usize::try_from(approx_tokens_from_byte_count_i64(estimate_image_bytes(
+ image_url, *detail,
+ )))
.unwrap_or(usize::MAX),
ContentItem::InputAudio { .. } => 0,
}
diff --git a/codex-rs/core/src/compact_tests.rs b/codex-rs/core/src/compact_tests.rs
index b319e75ddb..531d093061 100644
--- a/codex-rs/core/src/compact_tests.rs
+++ b/codex-rs/core/src/compact_tests.rs
@@ -10,6 +10,7 @@ use codex_protocol::models::ContentItemKind;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::ExecutedToolCall;
use codex_protocol::models::FunctionCallOutputPayload;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::InternalChatMessageMetadataPassthrough;
use core_test_support::responses;
use core_test_support::skip_if_no_network;
@@ -207,7 +208,9 @@ fn content_items_to_text_joins_non_empty_segments() {
#[test]
fn content_items_to_text_ignores_image_only_content() {
let items = vec![ContentItem::InputImage {
- image_url: "file://image.png".to_string(),
+ image: ImageReference::Inline {
+ image_url: "file://image.png".to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
}];
diff --git a/codex-rs/core/src/context/node_repl_review_evidence.rs b/codex-rs/core/src/context/node_repl_review_evidence.rs
index fdc2027da2..16cf5f62d1 100644
--- a/codex-rs/core/src/context/node_repl_review_evidence.rs
+++ b/codex-rs/core/src/context/node_repl_review_evidence.rs
@@ -12,6 +12,7 @@ use codex_guardian_context::NodeReplContext;
use codex_guardian_context::NodeReplResponse;
pub use codex_guardian_context::NodeReplReviewEvidenceMode;
use codex_protocol::models::ContentItem;
+use codex_protocol::models::ImageReference;
use codex_protocol::user_input::UserInput;
use codex_protocol::user_input::UserInput::Image;
use codex_protocol::user_input::UserInput::Text;
@@ -117,7 +118,9 @@ impl NodeReplReviewEvidence {
.filter_map(|item| match item {
Image { image_url, detail } if seen_images.insert(image_url) => {
Some(ContentItem::InputImage {
- image_url: image_url.clone(),
+ image: ImageReference::Inline {
+ image_url: image_url.clone(),
+ },
detail: *detail,
})
}
diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs
index 3695fc7b77..4617e462ab 100644
--- a/codex-rs/core/src/context_manager/history.rs
+++ b/codex-rs/core/src/context_manager/history.rs
@@ -43,6 +43,7 @@ use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::InputModality;
use codex_protocol::protocol::InterAgentCommunication;
@@ -850,9 +851,10 @@ fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
ContentItem::InputText { text } | ContentItem::OutputText { text } => {
text_bytes(text)
}
- ContentItem::InputImage { image_url, detail } => {
- estimate_image_bytes(image_url, *detail)
- }
+ ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail,
+ } => estimate_image_bytes(image_url, *detail),
ContentItem::InputAudio { audio_url } => estimate_audio_bytes(audio_url),
})
.fold(0i64, i64::saturating_add),
@@ -1055,9 +1057,10 @@ fn estimate_function_output_bytes(output: &FunctionCallOutputBody) -> i64 {
.iter()
.map(|part| match part {
FunctionCallOutputContentItem::InputText { text } => text_bytes(text),
- FunctionCallOutputContentItem::InputImage { image_url, detail } => {
- estimate_image_bytes(image_url, *detail)
- }
+ FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail,
+ } => estimate_image_bytes(image_url, *detail),
FunctionCallOutputContentItem::InputAudio { audio_url } => {
estimate_audio_bytes(audio_url)
}
diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs
index c0f35668a9..e0844511bd 100644
--- a/codex-rs/core/src/context_manager/history_tests.rs
+++ b/codex-rs/core/src/context_manager/history_tests.rs
@@ -18,6 +18,7 @@ use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::InternalChatMessageMetadataPassthrough;
use codex_protocol::models::LocalShellAction;
use codex_protocol::models::LocalShellExecAction;
@@ -715,7 +716,9 @@ fn for_prompt_annotated_preserves_metadata_while_normalizing_item() {
text: "keep".to_string(),
},
ContentItem::InputImage {
- image_url: "data:image/png;base64,abc".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,abc".to_string(),
+ },
detail: None,
},
],
@@ -796,7 +799,9 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {
text: "look at this".to_string(),
},
ContentItem::InputImage {
- image_url: "https://example.com/img.png".to_string(),
+ image: ImageReference::Inline {
+ image_url: "https://example.com/img.png".to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputAudio {
@@ -838,7 +843,9 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {
text: "image result".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "https://example.com/result.png".to_string(),
+ image: ImageReference::Inline {
+ image_url: "https://example.com/result.png".to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
@@ -865,7 +872,9 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {
text: "js repl result".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "https://example.com/js-repl-result.png".to_string(),
+ image: ImageReference::Inline {
+ image_url: "https://example.com/js-repl-result.png".to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
@@ -991,7 +1000,9 @@ fn for_prompt_strips_media_when_model_does_not_support_it() {
text: "look".to_string(),
},
ContentItem::InputImage {
- image_url: "https://example.com/img.png".to_string(),
+ image: ImageReference::Inline {
+ image_url: "https://example.com/img.png".to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
],
@@ -2414,7 +2425,7 @@ fn image_data_url_payload_does_not_dominate_message_estimate() {
text: "Here is the screenshot".to_string(),
},
ContentItem::InputImage {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
],
@@ -2455,7 +2466,7 @@ fn image_data_url_payload_does_not_dominate_function_call_output_estimate() {
text: "Screenshot captured".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]),
@@ -2484,7 +2495,7 @@ fn image_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {
text: "Screenshot captured".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]),
@@ -2628,7 +2639,9 @@ fn non_base64_image_urls_use_image_estimates() {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage {
- image_url: "https://example.com/foo.png".to_string(),
+ image: ImageReference::Inline {
+ image_url: "https://example.com/foo.png".to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
}],
phase: None,
@@ -2641,7 +2654,9 @@ fn non_base64_image_urls_use_image_estimates() {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
- image_url: "file:///tmp/foo.png".to_string(),
+ image: ImageReference::Inline {
+ image_url: "file:///tmp/foo.png".to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]),
@@ -2746,7 +2761,10 @@ fn data_url_without_base64_marker_uses_image_estimate() {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage {
- image_url: "data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg'/>".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg'/>"
+ .to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
}],
phase: None,
@@ -2771,7 +2789,7 @@ fn non_image_base64_data_url_uses_image_estimate() {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]),
@@ -2791,7 +2809,7 @@ fn mixed_case_data_url_markers_are_adjusted() {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
}],
phase: None,
@@ -2818,11 +2836,15 @@ fn multiple_inline_images_apply_multiple_fixed_costs() {
text: "images".to_string(),
},
ContentItem::InputImage {
- image_url: image_url_one,
+ image: ImageReference::Inline {
+ image_url: image_url_one,
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputImage {
- image_url: image_url_two,
+ image: ImageReference::Inline {
+ image_url: image_url_two,
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
],
@@ -2858,7 +2880,7 @@ fn original_detail_images_scale_with_dimensions() {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: Some(ImageDetail::Original),
},
]),
@@ -2891,7 +2913,7 @@ fn original_detail_images_are_capped_at_max_patch_count() {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: Some(ImageDetail::Original),
},
]),
@@ -2927,7 +2949,7 @@ fn original_detail_webp_images_scale_with_dimensions() {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: Some(ImageDetail::Original),
},
]),
diff --git a/codex-rs/core/src/event_mapping.rs b/codex-rs/core/src/event_mapping.rs
index d386b08b5d..ed983fe644 100644
--- a/codex-rs/core/src/event_mapping.rs
+++ b/codex-rs/core/src/event_mapping.rs
@@ -5,6 +5,7 @@ use codex_protocol::items::TurnItem;
use codex_protocol::items::UserMessageItem;
use codex_protocol::items::WebSearchItem;
use codex_protocol::models::ContentItem;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::MessagePhase;
use codex_protocol::models::ReasoningItemContent;
use codex_protocol::models::ReasoningItemReasoningSummary;
@@ -126,7 +127,10 @@ fn parse_user_message(message: &[ContentItem]) -> Option<UserMessageItem> {
text_elements: Vec::new(),
});
}
- ContentItem::InputImage { image_url, detail } => {
+ ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail,
+ } => {
content.push(UserInput::Image {
image_url: image_url.clone(),
detail: *detail,
diff --git a/codex-rs/core/src/event_mapping_tests.rs b/codex-rs/core/src/event_mapping_tests.rs
index 972df5e400..b6ac81c9da 100644
--- a/codex-rs/core/src/event_mapping_tests.rs
+++ b/codex-rs/core/src/event_mapping_tests.rs
@@ -12,6 +12,7 @@ use codex_protocol::items::WebSearchItem;
use codex_protocol::items::build_hook_prompt_message;
use codex_protocol::models::ContentItem;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ReasoningItemContent;
use codex_protocol::models::ReasoningItemReasoningSummary;
use codex_protocol::models::ResponseItem;
@@ -83,11 +84,15 @@ fn parses_user_message_with_text_and_two_images() {
text: "Hello world".to_string(),
},
ContentItem::InputImage {
- image_url: img1.clone(),
+ image: ImageReference::Inline {
+ image_url: img1.clone(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputImage {
- image_url: img2.clone(),
+ image: ImageReference::Inline {
+ image_url: img2.clone(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
],
@@ -131,7 +136,9 @@ fn skips_local_image_label_text() {
content: vec![
ContentItem::InputText { text: label },
ContentItem::InputImage {
- image_url: image_url.clone(),
+ image: ImageReference::Inline {
+ image_url: image_url.clone(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputText {
@@ -258,7 +265,9 @@ fn skips_unnamed_image_label_text() {
content: vec![
ContentItem::InputText { text: label },
ContentItem::InputImage {
- image_url: image_url.clone(),
+ image: ImageReference::Inline {
+ image_url: image_url.clone(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputText {
diff --git a/codex-rs/core/src/guardian/review_session.rs b/codex-rs/core/src/guardian/review_session.rs
index 23e756fcf4..397f1eb635 100644
--- a/codex-rs/core/src/guardian/review_session.rs
+++ b/codex-rs/core/src/guardian/review_session.rs
@@ -35,6 +35,7 @@ use codex_protocol::items::TurnItem;
use codex_protocol::mcp::is_node_repl_backed_server;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::InputModality;
@@ -502,7 +503,7 @@ async fn run_review_on_session(
_ => &[],
})
.filter_map(|item| match item {
- ContentItem::InputImage { image_url, .. } => Some(image_url.as_str()),
+ ContentItem::InputImage { image: ImageReference::Inline { image_url }, .. } => Some(image_url.as_str()),
_ => None,
})
.collect::<HashSet<_>>();
@@ -550,7 +551,7 @@ async fn run_review_on_session(
return false;
};
content.iter().any(|item| {
- matches!(item, ContentItem::InputImage { image_url, .. }
+ matches!(item, ContentItem::InputImage { image: ImageReference::Inline { image_url }, .. }
if !reviewer_image_urls.contains(image_url.as_str()))
})
});
diff --git a/codex-rs/core/src/guardian/tests.rs b/codex-rs/core/src/guardian/tests.rs
index 777b77c54a..cf191851bf 100644
--- a/codex-rs/core/src/guardian/tests.rs
+++ b/codex-rs/core/src/guardian/tests.rs
@@ -45,6 +45,7 @@ use codex_protocol::config_types::ApprovalsReviewer;
use codex_protocol::config_types::ReasoningSummary;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::ResponseItem;
use codex_protocol::models::SandboxPermissions;
@@ -1130,7 +1131,9 @@ fn collect_guardian_transcript_entries_preserves_named_unpaired_tool_sources() {
if let ResponseItem::FunctionCallOutput { output, .. } = &mut items[0] {
*output = codex_protocol::models::FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,image".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,image".to_string(),
+ },
detail: None,
},
]);
diff --git a/codex-rs/core/src/image_preparation.rs b/codex-rs/core/src/image_preparation.rs
index 78170db319..adc886f7c1 100644
--- a/codex-rs/core/src/image_preparation.rs
+++ b/codex-rs/core/src/image_preparation.rs
@@ -15,6 +15,7 @@ use codex_protocol::models::ContentItemKind;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ModelInfo;
use codex_utils_image::ImageProcessingError;
@@ -185,7 +186,11 @@ fn prepare_message_content(
let mut image_number = 0;
let mut resized_images = Vec::new();
for item in items {
- if let ContentItem::InputImage { image_url, detail } = item.content_mut() {
+ if let ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail,
+ } = item.content_mut()
+ {
image_number += 1;
match prepare_image(image_url, detail, origin, metadata, mode) {
Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => {
@@ -226,7 +231,11 @@ fn prepare_tool_output_content(
let mut image_number = 0;
let mut resized_images = Vec::new();
for item in items {
- if let FunctionCallOutputContentItem::InputImage { image_url, detail } = item {
+ if let FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail,
+ } = item
+ {
image_number += 1;
match prepare_image(image_url, detail, origin, metadata, mode) {
Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => {
diff --git a/codex-rs/core/src/image_preparation_tests.rs b/codex-rs/core/src/image_preparation_tests.rs
index 67b1f1d4df..a0cfbf5479 100644
--- a/codex-rs/core/src/image_preparation_tests.rs
+++ b/codex-rs/core/src/image_preparation_tests.rs
@@ -5,6 +5,7 @@ use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use codex_protocol::models::ContentItemKind;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputPayload;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::InternalChatMessageMetadataPassthrough;
use codex_utils_image::data_url_from_bytes;
use image::DynamicImage;
@@ -41,11 +42,15 @@ fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() {
role: "user".to_string(),
content: vec![
ContentItem::InputImage {
- image_url: data_url,
+ image: ImageReference::Inline {
+ image_url: data_url,
+ },
detail: Some(ImageDetail::High),
},
ContentItem::InputImage {
- image_url: "https://example.com/image.png".to_string(),
+ image: ImageReference::Inline {
+ image_url: "https://example.com/image.png".to_string(),
+ },
detail: Some(ImageDetail::Low),
},
],
@@ -56,7 +61,9 @@ fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() {
id: None,
role: "developer".to_string(),
content: vec![ContentItem::InputImage {
- image_url: "https://example.com/developer-image.png".to_string(),
+ image: ImageReference::Inline {
+ image_url: "https://example.com/developer-image.png".to_string(),
+ },
detail: Some(ImageDetail::High),
}],
phase: None,
@@ -73,7 +80,10 @@ fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() {
panic!("expected message");
};
let [
- ContentItem::InputImage { image_url, .. },
+ ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ ..
+ },
ContentItem::InputText { text },
] = content.as_slice()
else {
@@ -135,7 +145,10 @@ fn detail_policies_apply_the_expected_budgets() {
let mut items = vec![ResponseItem::Message {
id: None,
role: "user".to_string(),
- content: vec![ContentItem::InputImage { image_url, detail }],
+ content: vec![ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail,
+ }],
phase: None,
internal_chat_message_metadata_passthrough: None,
}];
@@ -149,7 +162,13 @@ fn detail_policies_apply_the_expected_budgets() {
let ResponseItem::Message { content, .. } = &items[0] else {
panic!("expected message");
};
- let [ContentItem::InputImage { image_url, .. }] = content.as_slice() else {
+ let [
+ ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ ..
+ },
+ ] = content.as_slice()
+ else {
panic!("expected image");
};
assert_eq!(decoded_image(image_url).1.dimensions(), expected_dimensions);
@@ -179,7 +198,7 @@ fn preparation_reports_tool_output_item_id() {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: Some(ImageDetail::High),
},
]),
@@ -215,15 +234,21 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
role: "user".to_string(),
content: vec![
ContentItem::InputImage {
- image_url: small_image_url,
+ image: ImageReference::Inline {
+ image_url: small_image_url,
+ },
detail: Some(ImageDetail::High),
},
ContentItem::InputImage {
- image_url: "data:image/png;base64,%%%".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,%%%".to_string(),
+ },
detail: Some(ImageDetail::High),
},
ContentItem::InputImage {
- image_url: large_image_url.clone(),
+ image: ImageReference::Inline {
+ image_url: large_image_url.clone(),
+ },
detail: Some(ImageDetail::High),
},
],
@@ -246,11 +271,15 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,%%%".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,%%%".to_string(),
+ },
detail: Some(ImageDetail::High),
},
FunctionCallOutputContentItem::InputImage {
- image_url: large_image_url,
+ image: ImageReference::Inline {
+ image_url: large_image_url,
+ },
detail: Some(ImageDetail::High),
},
]),
@@ -290,14 +319,20 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
);
let [
ContentItem::InputImage {
- image_url: small_message_image_url,
+ image:
+ ImageReference::Inline {
+ image_url: small_message_image_url,
+ },
..
},
ContentItem::InputText {
text: failed_message_image,
},
ContentItem::InputImage {
- image_url: resized_message_image_url,
+ image:
+ ImageReference::Inline {
+ image_url: resized_message_image_url,
+ },
..
},
] = content.as_slice()
@@ -342,7 +377,10 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
text: failed_tool_image,
},
FunctionCallOutputContentItem::InputImage {
- image_url: resized_tool_image_url,
+ image:
+ ImageReference::Inline {
+ image_url: resized_tool_image_url,
+ },
..
},
] = output.content_items().expect("tool output content items")
@@ -394,19 +432,27 @@ fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() {
text: "before".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,%%%".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,%%%".to_string(),
+ },
detail: Some(ImageDetail::High),
},
FunctionCallOutputContentItem::InputImage {
- image_url: data_url_from_bytes("image/png", b"not an image"),
+ image: ImageReference::Inline {
+ image_url: data_url_from_bytes("image/png", b"not an image"),
+ },
detail: Some(ImageDetail::High),
},
FunctionCallOutputContentItem::InputImage {
- image_url: valid_image_url.clone(),
+ image: ImageReference::Inline {
+ image_url: valid_image_url.clone(),
+ },
detail: Some(ImageDetail::Low),
},
FunctionCallOutputContentItem::InputImage {
- image_url: valid_image_url,
+ image: ImageReference::Inline {
+ image_url: valid_image_url,
+ },
detail: Some(ImageDetail::High),
},
]),
@@ -442,7 +488,9 @@ fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() {
text: UNSUPPORTED_LOW_DETAIL_PLACEHOLDER.to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: expected_valid_image_url,
+ image: ImageReference::Inline {
+ image_url: expected_valid_image_url
+ },
detail: Some(ImageDetail::High),
},
]),
diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs
index f9adf42a67..56ca71705d 100644
--- a/codex-rs/core/src/session/tests.rs
+++ b/codex-rs/core/src/session/tests.rs
@@ -74,6 +74,7 @@ use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::SandboxEnforcement;
use codex_protocol::openai_models::ModelServiceTier;
@@ -2575,11 +2576,15 @@ async fn prepares_image_failures_before_history_insertion() {
text: "before".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,%%%".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,%%%".to_string(),
+ },
detail: Some(ImageDetail::High),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "https://example.com/image.png".to_string(),
+ image: ImageReference::Inline {
+ image_url: "https://example.com/image.png".to_string(),
+ },
detail: Some(ImageDetail::High),
},
]),
@@ -2644,11 +2649,15 @@ async fn prepares_resumed_history_before_installing_it() {
role: "user".to_string(),
content: vec![
ContentItem::InputImage {
- image_url: "data:image/png;base64,%%%".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,%%%".to_string(),
+ },
detail: Some(ImageDetail::High),
},
ContentItem::InputImage {
- image_url: "https://example.com/image.png".to_string(),
+ image: ImageReference::Inline {
+ image_url: "https://example.com/image.png".to_string(),
+ },
detail: Some(ImageDetail::High),
},
ContentItem::InputText {
@@ -11270,7 +11279,9 @@ async fn task_finish_emits_turn_item_lifecycle_for_leftover_pending_user_input()
text: "late pending input".to_string(),
},
ContentItem::InputImage {
- image_url: image_url.clone(),
+ image: ImageReference::Inline {
+ image_url: image_url.clone(),
+ },
detail: Some(ImageDetail::Original),
},
],
diff --git a/codex-rs/core/src/tools/code_mode/response_adapter.rs b/codex-rs/core/src/tools/code_mode/response_adapter.rs
index 635eb94900..f28d5da934 100644
--- a/codex-rs/core/src/tools/code_mode/response_adapter.rs
+++ b/codex-rs/core/src/tools/code_mode/response_adapter.rs
@@ -2,6 +2,7 @@ use codex_code_mode::ImageDetail as CodeModeImageDetail;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
trait IntoProtocol<T> {
fn into_protocol(self) -> T;
@@ -36,7 +37,7 @@ impl IntoProtocol<FunctionCallOutputContentItem>
}
codex_code_mode::FunctionCallOutputContentItem::InputImage { image_url, detail } => {
FunctionCallOutputContentItem::InputImage {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: detail
.map(IntoProtocol::into_protocol)
.or(Some(DEFAULT_IMAGE_DETAIL)),
diff --git a/codex-rs/core/src/tools/context_tests.rs b/codex-rs/core/src/tools/context_tests.rs
index 7a393d35d7..bd0cb0c5b5 100644
--- a/codex-rs/core/src/tools/context_tests.rs
+++ b/codex-rs/core/src/tools/context_tests.rs
@@ -1,5 +1,6 @@
use super::*;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::SearchToolCallParams;
use core_test_support::assert_regex_match;
use pretty_assertions::assert_eq;
@@ -217,7 +218,9 @@ fn mcp_tool_output_response_item_preserves_content_items() {
text: "Wall time: 0.5000 seconds\nOutput:".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: image_url.to_string(),
+ image: ImageReference::Inline {
+ image_url: image_url.to_string()
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]
@@ -295,7 +298,9 @@ fn custom_tool_calls_can_derive_text_from_content_items() {
text: "line 1".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,AAA".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,AAA".to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputText {
@@ -315,7 +320,9 @@ fn custom_tool_calls_can_derive_text_from_content_items() {
text: "line 1".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,AAA".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,AAA".to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputText {
diff --git a/codex-rs/core/src/tools/handlers/view_image.rs b/codex-rs/core/src/tools/handlers/view_image.rs
index e447208fd4..fd4e3910db 100644
--- a/codex-rs/core/src/tools/handlers/view_image.rs
+++ b/codex-rs/core/src/tools/handlers/view_image.rs
@@ -7,6 +7,7 @@ use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::openai_models::InputModality;
use codex_utils_image::data_url_from_bytes;
@@ -238,7 +239,9 @@ impl ToolOutput for ViewImageOutput {
fn to_response_item(&self, call_id: &str, _payload: &ToolPayload) -> ResponseInputItem {
let body =
FunctionCallOutputBody::ContentItems(vec![FunctionCallOutputContentItem::InputImage {
- image_url: self.image_url.clone(),
+ image: ImageReference::Inline {
+ image_url: self.image_url.clone(),
+ },
detail: Some(self.image_detail),
}]);
let output = FunctionCallOutputPayload {
diff --git a/codex-rs/core/tests/suite/client.rs b/codex-rs/core/tests/suite/client.rs
index 033126a16e..79d0eb4f6d 100644
--- a/codex-rs/core/tests/suite/client.rs
+++ b/codex-rs/core/tests/suite/client.rs
@@ -40,6 +40,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::LocalShellAction;
use codex_protocol::models::LocalShellExecAction;
use codex_protocol::models::LocalShellStatus;
@@ -1078,7 +1079,9 @@ async fn resume_replays_legacy_js_repl_image_rollout_shapes() {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage {
- image_url: legacy_image_url.to_string(),
+ image: ImageReference::Inline {
+ image_url: legacy_image_url.to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
}],
phase: None,
@@ -1218,7 +1221,9 @@ async fn resume_replays_image_tool_outputs_with_detail() {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
- image_url: image_url.to_string(),
+ image: ImageReference::Inline {
+ image_url: image_url.to_string(),
+ },
detail: Some(ImageDetail::Original),
},
]),
@@ -1247,7 +1252,9 @@ async fn resume_replays_image_tool_outputs_with_detail() {
name: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
- image_url: image_url.to_string(),
+ image: ImageReference::Inline {
+ image_url: image_url.to_string(),
+ },
detail: Some(ImageDetail::Original),
},
]),
diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs
index 1663fa0a1a..acb1f4b26e 100644
--- a/codex-rs/core/tests/suite/code_mode.rs
+++ b/codex-rs/core/tests/suite/code_mode.rs
@@ -42,6 +42,7 @@ use codex_protocol::dynamic_tools::DynamicToolResponse;
use codex_protocol::dynamic_tools::DynamicToolSpec;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::InputModality;
@@ -5662,7 +5663,9 @@ async fn code_mode_node_repl_screenshots_can_be_captured_without_guardian_transc
assert_eq!(
evidence.images(),
vec![ContentItem::InputImage {
- image_url: SCREENSHOT.to_owned(),
+ image: ImageReference::Inline {
+ image_url: SCREENSHOT.to_owned()
+ },
detail: Some(ImageDetail::Low),
}]
);
diff --git a/codex-rs/core/tests/suite/image_rollout.rs b/codex-rs/core/tests/suite/image_rollout.rs
index 3316742feb..e12d6a7f57 100644
--- a/codex-rs/core/tests/suite/image_rollout.rs
+++ b/codex-rs/core/tests/suite/image_rollout.rs
@@ -10,6 +10,7 @@ use codex_protocol::config_types::Settings;
use codex_protocol::models::ContentItem;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ResponseItem;
@@ -69,7 +70,10 @@ fn find_user_message_with_image(text: &str) -> Option<ResponseItem> {
fn extract_image_url(item: &ResponseItem) -> Option<String> {
match item {
ResponseItem::Message { content, .. } => content.iter().find_map(|span| match span {
- ContentItem::InputImage { image_url, .. } => Some(image_url.clone()),
+ ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ ..
+ } => Some(image_url.clone()),
_ => None,
}),
_ => None,
@@ -178,7 +182,7 @@ async fn copy_paste_local_image_persists_rollout_request_shape() -> anyhow::Resu
),
},
ContentItem::InputImage {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputText {
@@ -269,7 +273,7 @@ async fn drag_drop_image_persists_rollout_request_shape() -> anyhow::Result<()>
role: "user".to_string(),
content: vec![
ContentItem::InputImage {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
ContentItem::InputText {
@@ -341,7 +345,9 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R
historical_content.insert(
/*index*/ 0,
ContentItem::InputImage {
- image_url: original_image_url.clone(),
+ image: ImageReference::Inline {
+ image_url: original_image_url.clone(),
+ },
detail: Some(ImageDetail::High),
},
);
@@ -480,7 +486,9 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R
ResponseInputItem::Message {
role: "user".to_string(),
content: vec![ContentItem::InputImage {
- image_url: original_image_url,
+ image: ImageReference::Inline {
+ image_url: original_image_url,
+ },
detail: Some(ImageDetail::High),
}],
phase: None,
diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs
index 78999fca06..23405bf9fc 100644
--- a/codex-rs/core/tests/suite/rmcp_client.rs
+++ b/codex-rs/core/tests/suite/rmcp_client.rs
@@ -52,6 +52,7 @@ use codex_protocol::mcp_policy::McpServerRequirement;
use codex_protocol::mcp_policy::PluginMcpRequirements;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::PermissionProfileSnapshot;
use codex_protocol::models::ResponseItem;
@@ -1549,7 +1550,9 @@ async fn interrupt_during_mcp_startup_preserves_user_input_in_history(
};
assert!(
content.contains(&ContentItem::InputImage {
- image_url: OPENAI_PNG.to_string(),
+ image: ImageReference::Inline {
+ image_url: OPENAI_PNG.to_string()
+ },
detail: Some(ImageDetail::Original),
}),
"interrupted input must use the current model's unified image budget"
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/budget_tests.rs b/codex-rs/ext/guardian-v2/src/async_scorer/budget_tests.rs
index 171dc00518..9454713ef8 100644
--- a/codex-rs/ext/guardian-v2/src/async_scorer/budget_tests.rs
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/budget_tests.rs
@@ -2,6 +2,7 @@
use super::*;
use crate::async_scorer::authorization::ScoreAuthorization;
+use codex_protocol::models::ImageReference;
use pretty_assertions::assert_eq;
#[derive(Clone, Copy)]
@@ -139,7 +140,7 @@ async fn assert_catalog_budget(evidence: BudgetEvidence) -> Result<()> {
unreachable!("user instruction is a message");
};
content.push(ContentItem::InputImage {
- image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGPgEpEDAABoAD1UCKP3AAAAAElFTkSuQmCC".to_owned(),
+ image: ImageReference::Inline { image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGPgEpEDAABoAD1UCKP3AAAAAElFTkSuQmCC".to_owned() },
detail: None,
});
(None, vec!["optional old commentary ".repeat(/*n*/ 500)])
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs b/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs
index 31aac3bed6..38440c4f6c 100644
--- a/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/extension_tests.rs
@@ -38,6 +38,7 @@ use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::InternalChatMessageMetadataPassthrough;
use codex_protocol::models::LocalShellAction;
use codex_protocol::models::LocalShellExecAction;
@@ -1641,7 +1642,9 @@ async fn contributor_includes_transcript_images_by_default() -> Result<()> {
text: "Review what is shown on screen.".to_owned(),
},
ContentItem::InputImage {
- image_url: user_image.to_owned(),
+ image: ImageReference::Inline {
+ image_url: user_image.to_owned(),
+ },
detail: Some(ImageDetail::High),
},
],
@@ -1667,7 +1670,9 @@ async fn contributor_includes_transcript_images_by_default() -> Result<()> {
text: "Screenshot captured.".to_owned(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: tool_image.to_owned(),
+ image: ImageReference::Inline {
+ image_url: tool_image.to_owned(),
+ },
detail: Some(ImageDetail::High),
},
]),
diff --git a/codex-rs/ext/guardian-v2/src/async_scorer/transcript_tests.rs b/codex-rs/ext/guardian-v2/src/async_scorer/transcript_tests.rs
index ed1bc9ab6a..a3add7fc0a 100644
--- a/codex-rs/ext/guardian-v2/src/async_scorer/transcript_tests.rs
+++ b/codex-rs/ext/guardian-v2/src/async_scorer/transcript_tests.rs
@@ -7,6 +7,7 @@ use codex_protocol::models::AgentMessageInputContent;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::MessagePhase;
use codex_protocol::models::ReasoningItemContent;
use codex_protocol::models::ReasoningItemReasoningSummary;
@@ -1077,7 +1078,9 @@ fn transcript_preserves_outputs_with_call_ids_or_explicit_names() {
if let ResponseItem::FunctionCallOutput { output, .. } = &mut items[0] {
*output = FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,image".to_owned(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,image".to_owned(),
+ },
detail: None,
},
]);
@@ -1222,7 +1225,9 @@ fn transcript_omits_media_payloads_and_keeps_readable_content() {
text: "Review this screenshot.".to_string(),
},
ContentItem::InputImage {
- image_url: format!("data:image/png;base64,{oversized_image}"),
+ image: ImageReference::Inline {
+ image_url: format!("data:image/png;base64,{oversized_image}"),
+ },
detail: None,
},
ContentItem::InputAudio {
@@ -1242,7 +1247,9 @@ fn transcript_omits_media_payloads_and_keeps_readable_content() {
text: "Screenshot captured.".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,tool-image".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,tool-image".to_string(),
+ },
detail: None,
},
FunctionCallOutputContentItem::InputAudio {
diff --git a/codex-rs/ext/history-notes/src/tools.rs b/codex-rs/ext/history-notes/src/tools.rs
index ad7422c961..114ee2ece8 100644
--- a/codex-rs/ext/history-notes/src/tools.rs
+++ b/codex-rs/ext/history-notes/src/tools.rs
@@ -11,6 +11,7 @@ use codex_extension_api::parse_tool_input_schema;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseInputItem;
use codex_tools::JsonToolOutput;
use codex_tools::ResponsesApiNamespace;
@@ -367,7 +368,9 @@ impl HistoryNotesToolOutput {
serde_json::from_value(image.get("detail").cloned().unwrap_or(Value::Null))
.map_err(|_| invalid_image())?;
content.push(FunctionCallOutputContentItem::InputImage {
- image_url: format!("data:{mime_type};base64,{data}"),
+ image: ImageReference::Inline {
+ image_url: format!("data:{mime_type};base64,{data}"),
+ },
detail,
});
}
diff --git a/codex-rs/ext/history-notes/src/tools_tests.rs b/codex-rs/ext/history-notes/src/tools_tests.rs
index 55b99cdc35..9ec63f6621 100644
--- a/codex-rs/ext/history-notes/src/tools_tests.rs
+++ b/codex-rs/ext/history-notes/src/tools_tests.rs
@@ -1,6 +1,7 @@
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseInputItem;
use codex_tools::ToolOutput;
use codex_tools::ToolPayload;
@@ -73,19 +74,27 @@ fn preserves_images_as_separate_output_items_without_logging_bytes() {
encrypted_content: "enc_payload".to_string()
},
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,cG5n".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,cG5n".to_string()
+ },
detail: Some(ImageDetail::Original)
},
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/jpeg;base64,anBlZw==".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/jpeg;base64,anBlZw==".to_string()
+ },
detail: Some(ImageDetail::Low)
},
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/gif;base64,Z2lm".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/gif;base64,Z2lm".to_string()
+ },
detail: None
},
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/webp;base64,d2VicA==".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/webp;base64,d2VicA==".to_string()
+ },
detail: None
},
])
diff --git a/codex-rs/ext/image-generation/src/tests.rs b/codex-rs/ext/image-generation/src/tests.rs
index c7bca61e8a..32b0493775 100644
--- a/codex-rs/ext/image-generation/src/tests.rs
+++ b/codex-rs/ext/image-generation/src/tests.rs
@@ -12,6 +12,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ResponseItem;
use codex_tools::ResponsesApiNamespaceTool;
@@ -266,7 +267,9 @@ fn generated_output_returns_image_input_and_output_hint() {
content_items,
vec![
FunctionCallOutputContentItem::InputImage {
- image_url: format!("data:image/png;base64,{RESULT}"),
+ image: ImageReference::Inline {
+ image_url: format!("data:image/png;base64,{RESULT}")
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputText { text: output_hint },
@@ -311,7 +314,9 @@ fn generated_output_omits_oversized_output_hint() {
assert_eq!(
content_items,
vec![FunctionCallOutputContentItem::InputImage {
- image_url: format!("data:image/png;base64,{RESULT}"),
+ image: ImageReference::Inline {
+ image_url: format!("data:image/png;base64,{RESULT}")
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
}]
);
@@ -319,14 +324,18 @@ fn generated_output_omits_oversized_output_hint() {
fn input_image(image: &str) -> ContentItem {
ContentItem::InputImage {
- image_url: format!("data:image/png;base64,{image}"),
+ image: ImageReference::Inline {
+ image_url: format!("data:image/png;base64,{image}"),
+ },
detail: None,
}
}
fn image_output(image: &str) -> FunctionCallOutputPayload {
FunctionCallOutputPayload::from_content_items(vec![FunctionCallOutputContentItem::InputImage {
- image_url: format!("data:image/png;base64,{image}"),
+ image: ImageReference::Inline {
+ image_url: format!("data:image/png;base64,{image}"),
+ },
detail: None,
}])
}
diff --git a/codex-rs/ext/image-generation/src/tool.rs b/codex-rs/ext/image-generation/src/tool.rs
index ac36240cc4..8fc408b442 100644
--- a/codex-rs/ext/image-generation/src/tool.rs
+++ b/codex-rs/ext/image-generation/src/tool.rs
@@ -29,6 +29,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::EventMsg;
@@ -494,7 +495,10 @@ fn recent_images(history: &[ResponseItem], count: usize) -> Vec<ImageUrl> {
match item {
ResponseItem::Message { content, .. } => {
image_urls.extend(content.iter().rev().filter_map(|item| match item {
- ContentItem::InputImage { image_url, .. } => Some(image_url.clone()),
+ ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ ..
+ } => Some(image_url.clone()),
ContentItem::InputText { .. }
| ContentItem::InputAudio { .. }
| ContentItem::OutputText { .. } => None,
@@ -542,7 +546,10 @@ fn output_image_urls(output: &FunctionCallOutputPayload) -> impl Iterator<Item =
.flatten()
.rev()
.filter_map(|item| match item {
- FunctionCallOutputContentItem::InputImage { image_url, .. } => Some(image_url.clone()),
+ FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ ..
+ } => Some(image_url.clone()),
FunctionCallOutputContentItem::InputText { .. }
| FunctionCallOutputContentItem::InputAudio { .. }
| FunctionCallOutputContentItem::EncryptedContent { .. } => None,
@@ -651,7 +658,9 @@ impl ToolOutput for GeneratedImageOutput {
/// Returns generated bytes and persisted-artifact context for model follow-up.
fn to_response_item(&self, call_id: &str, _payload: &ToolPayload) -> ResponseInputItem {
let mut content = vec![FunctionCallOutputContentItem::InputImage {
- image_url: format!("data:image/png;base64,{}", self.result),
+ image: ImageReference::Inline {
+ image_url: format!("data:image/png;base64,{}", self.result),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
}];
if let Some(output_hint) = &self.output_hint {
diff --git a/codex-rs/ext/web-search/src/history.rs b/codex-rs/ext/web-search/src/history.rs
index fb1630c8dc..6d64e22124 100644
--- a/codex-rs/ext/web-search/src/history.rs
+++ b/codex-rs/ext/web-search/src/history.rs
@@ -84,6 +84,7 @@ mod tests {
use codex_api::SearchInput;
use codex_protocol::ResponseItemId;
use codex_protocol::models::ContentItem;
+ use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use pretty_assertions::assert_eq;
@@ -158,7 +159,9 @@ mod tests {
text: "previous user".to_string(),
},
ContentItem::InputImage {
- image_url: "data:image/png;base64,image".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,image".to_string(),
+ },
detail: None,
},
],
diff --git a/codex-rs/guardian-context/src/budget.rs b/codex-rs/guardian-context/src/budget.rs
index 7f172a9c01..b29ff2c66f 100644
--- a/codex-rs/guardian-context/src/budget.rs
+++ b/codex-rs/guardian-context/src/budget.rs
@@ -5,6 +5,7 @@
use std::io::Write;
use codex_protocol::models::ContentItem;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::TruncationPolicy;
@@ -97,7 +98,10 @@ impl SectionCost {
ContentItem::InputText { text } | ContentItem::OutputText { text } => {
self.text_bytes = self.text_bytes.saturating_add(text.len());
}
- ContentItem::InputImage { image_url, .. } => {
+ ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ ..
+ } => {
self.image_bytes = self.image_bytes.saturating_add(image_url.len());
self.image_count = self.image_count.saturating_add(1);
}
@@ -206,7 +210,11 @@ pub(super) fn content_framing_tokens(item_count: usize) -> usize {
fn adjusted_tokens(mut bytes: usize, content: &[ContentItem]) -> usize {
for item in content {
- if let ContentItem::InputImage { image_url, .. } = item {
+ if let ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ ..
+ } = item
+ {
let payload = ByteCount::measure(|counter| serde_json::to_writer(counter, image_url));
if payload == usize::MAX {
return usize::MAX;
diff --git a/codex-rs/guardian-context/src/budget_tests.rs b/codex-rs/guardian-context/src/budget_tests.rs
index 5af8e7460a..33fc174ad7 100644
--- a/codex-rs/guardian-context/src/budget_tests.rs
+++ b/codex-rs/guardian-context/src/budget_tests.rs
@@ -4,6 +4,7 @@ use super::*;
use crate::Budgeted;
use crate::composition::SectionOutput;
use crate::composition::user_message as message;
+use codex_protocol::models::ImageReference;
use pretty_assertions::assert_eq;
#[test]
@@ -17,7 +18,9 @@ fn section_costs_keep_multimodal_payloads_separate() {
text: "évidence".to_owned(),
}),
Budgeted::required(ContentItem::InputImage {
- image_url: "data:image/png;base64,AAAA".to_owned(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,AAAA".to_owned(),
+ },
detail: None,
}),
]),
@@ -59,7 +62,9 @@ fn section_costs_keep_multimodal_payloads_separate() {
fn request_estimate_reserves_images_independently_of_encoded_size() {
let image = |payload: &str| {
message(vec![ContentItem::InputImage {
- image_url: format!("data:image/png;base64,{payload}"),
+ image: ImageReference::Inline {
+ image_url: format!("data:image/png;base64,{payload}"),
+ },
detail: None,
}])
};
diff --git a/codex-rs/guardian-context/src/composition.rs b/codex-rs/guardian-context/src/composition.rs
index 29c7427870..555fbb2163 100644
--- a/codex-rs/guardian-context/src/composition.rs
+++ b/codex-rs/guardian-context/src/composition.rs
@@ -6,6 +6,7 @@
use codex_context_fragments::ContextualUserFragment;
use codex_protocol::models::ContentItem;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::TruncationPolicy;
use codex_protocol::user_input::UserInput;
@@ -214,7 +215,10 @@ impl CollectedContext {
Ok(Budgeted::required(ContentItem::InputText { text }))
}
UserInput::Image { image_url, detail } => Ok(Budgeted::optional(
- ContentItem::InputImage { image_url, detail },
+ ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail,
+ },
BudgetPriority::Image,
)),
_ => Err(SectionError::UnsupportedDelivery {
@@ -281,9 +285,10 @@ impl ComposedContext {
}));
continue;
}
- ContentItem::InputImage { image_url, detail } => {
- UserInput::Image { image_url, detail }
- }
+ ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail,
+ } => UserInput::Image { image_url, detail },
ContentItem::InputAudio { .. } | ContentItem::OutputText { .. } => {
return Err(SectionError::UnsupportedDelivery {
section: section.id,
diff --git a/codex-rs/guardian-context/src/enforcement.rs b/codex-rs/guardian-context/src/enforcement.rs
index 99c88dff12..54cf00d076 100644
--- a/codex-rs/guardian-context/src/enforcement.rs
+++ b/codex-rs/guardian-context/src/enforcement.rs
@@ -7,6 +7,7 @@ use std::collections::HashSet;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::protocol::TruncationPolicy;
use crate::ComposedContext;
@@ -90,7 +91,10 @@ impl ComposedContext {
pub fn retain_images(&mut self, mut admit: impl FnMut(&str, &mut Option<ImageDetail>) -> bool) {
for section in &mut self.sections {
retain_content(section, &mut self.truncations, |_, item| match item {
- ContentItem::InputImage { image_url, detail } => admit(image_url, detail),
+ ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail,
+ } => admit(image_url, detail),
_ => true,
});
}
@@ -273,7 +277,10 @@ fn retain_content(
if !keep {
let original_bytes = match &item.content {
ContentItem::InputText { text } | ContentItem::OutputText { text } => text.len(),
- ContentItem::InputImage { image_url, .. } => image_url.len(),
+ ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ ..
+ } => image_url.len(),
ContentItem::InputAudio { audio_url } => audio_url.len(),
};
truncations.push(TruncationObservation {
diff --git a/codex-rs/guardian-context/src/enforcement_tests.rs b/codex-rs/guardian-context/src/enforcement_tests.rs
index 893e3fea64..88599eb177 100644
--- a/codex-rs/guardian-context/src/enforcement_tests.rs
+++ b/codex-rs/guardian-context/src/enforcement_tests.rs
@@ -3,6 +3,7 @@
use super::*;
use crate::budget::section_tokens;
use crate::composition::user_message;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use pretty_assertions::assert_eq;
@@ -146,7 +147,9 @@ fn budget_reserves_existing_context_and_preserves_required_messages() {
.unwrap()
.into_message();
let image = ContentItem::InputImage {
- image_url: "data:image/png;base64,AAAA".to_owned(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,AAAA".to_owned(),
+ },
detail: None,
};
let make_context = || ComposedContext {
@@ -241,7 +244,9 @@ fn image_omission_preserves_text_and_later_eviction_policy() {
delivery: SectionDelivery::UserContent(vec![
Budgeted::optional(
ContentItem::InputImage {
- image_url: "rejected-image".to_owned(),
+ image: ImageReference::Inline {
+ image_url: "rejected-image".to_owned(),
+ },
detail: None,
},
BudgetPriority::Image,
@@ -307,11 +312,15 @@ fn image_omission_preserves_text_and_later_eviction_policy() {
);
let older = ContentItem::InputImage {
- image_url: "older-image".to_owned(),
+ image: ImageReference::Inline {
+ image_url: "older-image".to_owned(),
+ },
detail: None,
};
let newer = ContentItem::InputImage {
- image_url: "newer-image".to_owned(),
+ image: ImageReference::Inline {
+ image_url: "newer-image".to_owned(),
+ },
detail: None,
};
let image_section = |images: Vec<ContentItem>| SectionOutput {
diff --git a/codex-rs/guardian-context/src/history_tests.rs b/codex-rs/guardian-context/src/history_tests.rs
index 4c03904ba2..f505003037 100644
--- a/codex-rs/guardian-context/src/history_tests.rs
+++ b/codex-rs/guardian-context/src/history_tests.rs
@@ -1,4 +1,5 @@
use codex_protocol::models::ExecutedToolCall;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ReasoningItemContent;
use pretty_assertions::assert_eq;
use serde_json::json;
@@ -147,7 +148,9 @@ fn oversized_user_images_preserve_text_and_metadata_in_order() {
content.insert(
/*index*/ 1,
ContentItem::InputImage {
- image_url: format!("data:image/png;base64,{}", "A".repeat(image_bytes)),
+ image: ImageReference::Inline {
+ image_url: format!("data:image/png;base64,{}", "A".repeat(image_bytes)),
+ },
detail: Some(codex_protocol::models::ImageDetail::Original),
},
);
diff --git a/codex-rs/guardian-context/src/images.rs b/codex-rs/guardian-context/src/images.rs
index 1d50b90930..ccae031095 100644
--- a/codex-rs/guardian-context/src/images.rs
+++ b/codex-rs/guardian-context/src/images.rs
@@ -10,6 +10,7 @@ use crate::SectionScope;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use std::collections::VecDeque;
@@ -65,7 +66,11 @@ impl TranscriptImages {
while images.len() >= MAX_TRANSCRIPT_IMAGES
|| image_bytes + image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES
{
- let Some(ContentItem::InputImage { image_url, .. }) = images.pop_front() else {
+ let Some(ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ ..
+ }) = images.pop_front()
+ else {
break;
};
image_bytes -= image_url.len();
@@ -73,7 +78,9 @@ impl TranscriptImages {
}
image_bytes += image_url.len();
images.push_back(ContentItem::InputImage {
- image_url: image_url.to_owned(),
+ image: ImageReference::Inline {
+ image_url: image_url.to_owned(),
+ },
detail,
});
};
@@ -84,7 +91,11 @@ impl TranscriptImages {
if matches!(role.as_str(), "user" | "assistant") =>
{
for item in content {
- if let ContentItem::InputImage { image_url, detail } = item {
+ if let ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail,
+ } = item
+ {
include_image(image_url, *detail);
}
}
@@ -95,8 +106,10 @@ impl TranscriptImages {
{
if let Some(content) = output.content_items() {
for item in content {
- if let FunctionCallOutputContentItem::InputImage { image_url, detail } =
- item
+ if let FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail,
+ } = item
{
include_image(image_url, *detail);
}
@@ -108,7 +121,11 @@ impl TranscriptImages {
}
if input.include_tool_outputs {
for image in input.node_repl_images {
- if let ContentItem::InputImage { image_url, detail } = image {
+ if let ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail,
+ } = image
+ {
include_image(image_url, *detail);
}
}
diff --git a/codex-rs/guardian-context/src/images_tests.rs b/codex-rs/guardian-context/src/images_tests.rs
index d39b7e8910..090b5c0531 100644
--- a/codex-rs/guardian-context/src/images_tests.rs
+++ b/codex-rs/guardian-context/src/images_tests.rs
@@ -5,12 +5,15 @@ use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use pretty_assertions::assert_eq;
fn image(url: &str) -> ContentItem {
ContentItem::InputImage {
- image_url: url.into(),
+ image: ImageReference::Inline {
+ image_url: url.into(),
+ },
detail: Some(ImageDetail::High),
}
}
@@ -32,7 +35,9 @@ fn image_selection_preserves_source_policy_order_and_both_limits() {
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
- image_url: "tool".into(),
+ image: ImageReference::Inline {
+ image_url: "tool".into(),
+ },
detail: Some(ImageDetail::High),
},
]),
diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs
index 7286f9865f..e2f588eaa3 100644
--- a/codex-rs/protocol/src/models.rs
+++ b/codex-rs/protocol/src/models.rs
@@ -879,7 +879,8 @@ pub enum ContentItem {
text: String,
},
InputImage {
- image_url: String,
+ #[serde(flatten)]
+ image: ImageReference,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional)]
detail: Option<ImageDetail>,
@@ -892,6 +893,13 @@ pub enum ContentItem {
},
}
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema, TS)]
+#[serde(untagged)]
+#[ts(untagged)]
+pub enum ImageReference {
+ Inline { image_url: String },
+}
+
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema, TS)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum AgentMessageInputContent {
@@ -1843,7 +1851,7 @@ fn local_image_content_items(
});
}
items.push(ContentItem::InputImage {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: Some(detail),
});
if label_number.is_some() {
@@ -2006,7 +2014,7 @@ impl ResponseInputItem {
image_index += 1;
let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL);
vec![ContentItem::InputImage {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: Some(detail),
}]
}
@@ -2080,7 +2088,8 @@ pub enum FunctionCallOutputContentItem {
},
// Do not rename, these are serialized and used directly in the responses API.
InputImage {
- image_url: String,
+ #[serde(flatten)]
+ image: ImageReference,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional)]
detail: Option<ImageDetail>,
@@ -2138,7 +2147,7 @@ impl From<crate::dynamic_tools::DynamicToolCallOutputContentItem>
}
crate::dynamic_tools::DynamicToolCallOutputContentItem::InputImage { image_url } => {
Self::InputImage {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
}
}
@@ -2383,7 +2392,7 @@ fn convert_mcp_content_to_items(
format!("data:{mime_type};base64,{data}")
};
FunctionCallOutputContentItem::InputImage {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: meta
.as_ref()
.and_then(serde_json::Value::as_object)
@@ -2656,7 +2665,9 @@ mod tests {
assert_eq!(
content_item,
ContentItem::InputImage {
- image_url: "data:image/png;base64,abc".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,abc".to_string(),
+ },
detail: Some(ImageDetail::Auto),
}
);
@@ -2711,7 +2722,9 @@ mod tests {
assert_eq!(
items,
vec![FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,Zm9v".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,Zm9v".to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
}]
);
@@ -3043,7 +3056,9 @@ mod tests {
assert_eq!(
items,
vec![FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,Zm9v".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,Zm9v".to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
}]
);
@@ -3100,7 +3115,9 @@ mod tests {
text: "line 1".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,AAA".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,AAA".to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputText {
@@ -3119,7 +3136,9 @@ mod tests {
text: " ".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,AAA".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,AAA".to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
@@ -3148,7 +3167,9 @@ mod tests {
text: "line 1".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,AAA".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,AAA".to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]);
@@ -3423,7 +3444,9 @@ mod tests {
text: "caption".into(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,BASE64".into(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,BASE64".into(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]
@@ -3498,7 +3521,9 @@ mod tests {
name: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,BASE64".into(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,BASE64".into(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]),
@@ -3563,7 +3588,9 @@ mod tests {
assert_eq!(
items,
vec![FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,BASE64".into(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,BASE64".into(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
}]
);
@@ -3595,7 +3622,9 @@ mod tests {
assert_eq!(
items,
vec![FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,BASE64".into(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,BASE64".into(),
+ },
detail: Some(ImageDetail::Original),
}]
);
@@ -3627,7 +3656,9 @@ mod tests {
assert_eq!(
items,
vec![FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,BASE64".into(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,BASE64".into(),
+ },
detail: Some(ImageDetail::High),
}]
);
@@ -3650,7 +3681,9 @@ mod tests {
text: "note".into(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,XYZ".into(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,XYZ".into(),
+ },
detail: None,
},
];
@@ -3859,7 +3892,7 @@ mod tests {
match item {
ResponseInputItem::Message { content, .. } => {
let expected = vec![ContentItem::InputImage {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
}];
assert_eq!(content, expected);
@@ -4005,7 +4038,7 @@ mod tests {
assert_eq!(
content.first(),
Some(&ContentItem::InputImage {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: Some(ImageDetail::Original),
})
);
@@ -4208,7 +4241,7 @@ mod tests {
assert_eq!(
content.first(),
Some(&ContentItem::InputImage {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
})
);
diff --git a/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs b/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs
index 588ac37787..07da7b7042 100644
--- a/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs
+++ b/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs
@@ -12,6 +12,7 @@ use std::collections::HashSet;
use codex_protocol::ResponseItemId;
use codex_protocol::models::ContentItem;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::UserMessageEvent;
@@ -492,7 +493,10 @@ fn user_response_matches_event(content: &[ContentItem], event: &UserMessageEvent
for item in content {
match item {
ContentItem::InputText { text: item_text } => text.push_str(item_text),
- ContentItem::InputImage { image_url, .. } => images.push(image_url.as_str()),
+ ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ ..
+ } => images.push(image_url.as_str()),
ContentItem::InputAudio { audio_url } => audio.push(audio_url.as_str()),
ContentItem::OutputText { .. } => return false,
}
diff --git a/codex-rs/tools/src/image_detail_tests.rs b/codex-rs/tools/src/image_detail_tests.rs
index 0fb26c10c4..c0949576fe 100644
--- a/codex-rs/tools/src/image_detail_tests.rs
+++ b/codex-rs/tools/src/image_detail_tests.rs
@@ -2,6 +2,7 @@ use super::*;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::openai_models::ModelInfo;
use pretty_assertions::assert_eq;
use serde_json::json;
@@ -88,11 +89,15 @@ fn sanitize_original_falls_back_to_high_without_support() {
text: "header".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,AAA".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,AAA".to_string(),
+ },
detail: Some(ImageDetail::Original),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,BBB".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,BBB".to_string(),
+ },
detail: Some(ImageDetail::Low),
},
];
@@ -106,11 +111,15 @@ fn sanitize_original_falls_back_to_high_without_support() {
text: "header".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,AAA".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,AAA".to_string()
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "data:image/png;base64,BBB".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,BBB".to_string()
+ },
detail: Some(ImageDetail::Low),
},
]
diff --git a/codex-rs/tools/src/tool_output.rs b/codex-rs/tools/src/tool_output.rs
index e95e662941..a299d172ee 100644
--- a/codex-rs/tools/src/tool_output.rs
+++ b/codex-rs/tools/src/tool_output.rs
@@ -2,6 +2,7 @@ use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseInputItem;
use serde_json::Value as JsonValue;
@@ -219,12 +220,13 @@ fn response_input_to_code_mode_result(response: ResponseInputItem) -> JsonValue
| codex_protocol::models::ContentItem::OutputText { text } => {
FunctionCallOutputContentItem::InputText { text }
}
- codex_protocol::models::ContentItem::InputImage { image_url, detail } => {
- FunctionCallOutputContentItem::InputImage {
- image_url,
- detail: detail.or(Some(DEFAULT_IMAGE_DETAIL)),
- }
- }
+ codex_protocol::models::ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail,
+ } => FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail: detail.or(Some(DEFAULT_IMAGE_DETAIL)),
+ },
codex_protocol::models::ContentItem::InputAudio { audio_url } => {
FunctionCallOutputContentItem::InputAudio { audio_url }
}
@@ -254,11 +256,10 @@ fn content_items_to_code_mode_result(items: &[FunctionCallOutputContentItem]) ->
FunctionCallOutputContentItem::InputText { text } if !text.trim().is_empty() => {
Some(text.clone())
}
- FunctionCallOutputContentItem::InputImage { image_url, .. }
- if !image_url.trim().is_empty() =>
- {
- Some(image_url.clone())
- }
+ FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ ..
+ } if !image_url.trim().is_empty() => Some(image_url.clone()),
FunctionCallOutputContentItem::InputAudio { audio_url }
if !audio_url.trim().is_empty() =>
{
diff --git a/codex-rs/utils/output-truncation/src/lib.rs b/codex-rs/utils/output-truncation/src/lib.rs
index f4f502ab28..da59e7b86d 100644
--- a/codex-rs/utils/output-truncation/src/lib.rs
+++ b/codex-rs/utils/output-truncation/src/lib.rs
@@ -3,6 +3,7 @@
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
+use codex_protocol::models::ImageReference;
pub use codex_utils_string::approx_bytes_for_tokens;
pub use codex_utils_string::approx_token_count;
pub use codex_utils_string::approx_tokens_from_byte_count;
@@ -90,12 +91,15 @@ pub fn formatted_truncate_text_content_items_with_policy(
text: formatted_truncate_text(&combined, policy),
}];
out.extend(items.iter().filter_map(|item| match item {
- FunctionCallOutputContentItem::InputImage { image_url, detail } => {
- Some(FunctionCallOutputContentItem::InputImage {
+ FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail,
+ } => Some(FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::Inline {
image_url: image_url.clone(),
- detail: *detail,
- })
- }
+ },
+ detail: *detail,
+ }),
FunctionCallOutputContentItem::InputAudio { audio_url } => {
Some(FunctionCallOutputContentItem::InputAudio {
audio_url: audio_url.clone(),
@@ -159,9 +163,14 @@ pub fn truncate_function_output_items_with_policy(
remaining_budget = 0;
}
}
- FunctionCallOutputContentItem::InputImage { image_url, detail } => {
+ FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail,
+ } => {
out.push(FunctionCallOutputContentItem::InputImage {
- image_url: image_url.clone(),
+ image: ImageReference::Inline {
+ image_url: image_url.clone(),
+ },
detail: *detail,
});
}
diff --git a/codex-rs/utils/output-truncation/src/truncate_tests.rs b/codex-rs/utils/output-truncation/src/truncate_tests.rs
index d58dace05d..bbcf617714 100644
--- a/codex-rs/utils/output-truncation/src/truncate_tests.rs
+++ b/codex-rs/utils/output-truncation/src/truncate_tests.rs
@@ -7,6 +7,7 @@ use crate::truncate_function_output_items_with_policy;
use crate::truncate_text;
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
use codex_protocol::models::FunctionCallOutputContentItem;
+use codex_protocol::models::ImageReference;
use pretty_assertions::assert_eq;
#[test]
@@ -114,7 +115,9 @@ fn truncates_across_multiple_under_limit_texts_and_reports_omitted() {
FunctionCallOutputContentItem::InputText { text: t1.clone() },
FunctionCallOutputContentItem::InputText { text: t2.clone() },
FunctionCallOutputContentItem::InputImage {
- image_url: "img:mid".to_string(),
+ image: ImageReference::Inline {
+ image_url: "img:mid".to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputText { text: t3 },
@@ -142,7 +145,9 @@ fn truncates_across_multiple_under_limit_texts_and_reports_omitted() {
assert_eq!(
output[2],
FunctionCallOutputContentItem::InputImage {
- image_url: "img:mid".to_string(),
+ image: ImageReference::Inline {
+ image_url: "img:mid".to_string()
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
}
);
@@ -183,7 +188,9 @@ fn truncate_function_output_items_with_policy_discards_empty_text() {
text: "caption".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "img:one".to_string(),
+ image: ImageReference::Inline {
+ image_url: "img:one".to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
@@ -253,7 +260,9 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_med
text: "abcd".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "img:one".to_string(),
+ image: ImageReference::Inline {
+ image_url: "img:one".to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputText {
@@ -266,7 +275,9 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_med
text: "ijkl".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "img:two".to_string(),
+ image: ImageReference::Inline {
+ image_url: "img:two".to_string(),
+ },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
];
@@ -281,14 +292,14 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_med
text: "Warning: truncated output (original token count: 4)\nTotal output lines: 3\n\nabcd…6 chars truncated…ijkl".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "img:one".to_string(),
+ image: ImageReference::Inline { image_url: "img:one".to_string() },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
FunctionCallOutputContentItem::InputAudio {
audio_url: "audio:one".to_string(),
},
FunctionCallOutputContentItem::InputImage {
- image_url: "img:two".to_string(),
+ image: ImageReference::Inline { image_url: "img:two".to_string() },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
]
diff --git a/sdk/python/scripts/update_sdk_artifacts.py b/sdk/python/scripts/update_sdk_artifacts.py
index 68234bfc88..61d98e29e4 100755
--- a/sdk/python/scripts/update_sdk_artifacts.py
+++ b/sdk/python/scripts/update_sdk_artifacts.py
@@ -575,6 +575,7 @@ def generate_v2_all(schema_dir: Path) -> None:
],
cwd=sdk_root(),
)
+ _preserve_inline_image_class_names(out_path)
_require_nullable_chatgpt_account_email(out_path)
_preserve_reasoning_effort_enum(out_path)
_preserve_thread_source_enum(out_path)
@@ -582,6 +583,23 @@ def generate_v2_all(schema_dir: Path) -> None:
_normalize_generated_timestamps(out_path)
+def _preserve_inline_image_class_names(out_path: Path) -> None:
+ """Keep the public class names used before ImageReference was introduced."""
+ source = out_path.read_text()
+ stable_names = {
+ "ImageUrlContentItem": "InputImageContentItem",
+ "ImageUrlFunctionCallOutputContentItem": "InputImageFunctionCallOutputContentItem",
+ }
+ for generated_name, stable_name in stable_names.items():
+ if source.count(f"class {generated_name}(") != 1:
+ raise RuntimeError(f"Generated SDK is missing a unique {generated_name} class")
+ if re.search(rf"\b{re.escape(stable_name)}\b", source):
+ raise RuntimeError(f"Generated SDK already defines {stable_name}")
+ source = re.sub(rf"\b{re.escape(generated_name)}\b", stable_name, source)
+
+ out_path.write_text(source)
+
+
def _require_nullable_chatgpt_account_email(out_path: Path) -> None:
"""Preserve required-but-nullable email semantics in the generated SDK model."""
source = out_path.read_text()
diff --git a/sdk/python/src/openai_codex/generated/v2_all.py b/sdk/python/src/openai_codex/generated/v2_all.py
index 621298ce6c..80839540a3 100644
--- a/sdk/python/src/openai_codex/generated/v2_all.py
+++ b/sdk/python/src/openai_codex/generated/v2_all.py
@@ -7784,8 +7784,8 @@ class InputImageContentItem(BaseModel):
populate_by_name=True,
)
detail: ImageDetail | None = None
- image_url: str
type: Annotated[Literal["input_image"], Field(title="InputImageContentItemType")]
+ image_url: str
class ContentItem(
@@ -7796,9 +7796,7 @@ class ContentItem(
model_config = ConfigDict(
populate_by_name=True,
)
- root: (
- InputTextContentItem | InputImageContentItem | InputAudioContentItem | OutputTextContentItem
- )
+ root: InputTextContentItem | InputImageContentItem | InputAudioContentItem | OutputTextContentItem
class ExperimentalFeature(BaseModel):
@@ -7980,10 +7978,10 @@ class InputImageFunctionCallOutputContentItem(BaseModel):
populate_by_name=True,
)
detail: ImageDetail | None = None
- image_url: str
type: Annotated[
Literal["input_image"], Field(title="InputImageFunctionCallOutputContentItemType")
]
+ image_url: str
class FunctionCallOutputContentItem(
diff --git a/sdk/python/tests/test_artifact_workflow_and_binaries.py b/sdk/python/tests/test_artifact_workflow_and_binaries.py
index d9c6617ed7..7fae0b7a57 100644
--- a/sdk/python/tests/test_artifact_workflow_and_binaries.py
+++ b/sdk/python/tests/test_artifact_workflow_and_binaries.py
@@ -536,6 +536,20 @@ def test_generated_chatgpt_account_email_is_required_nullable() -> None:
ChatgptAccount.model_validate({"planType": "pro", "type": "chatgpt"})
+def test_generated_inline_image_class_names_remain_stable() -> None:
+ """Keep the existing Python class names when image references expand."""
+ from openai_codex.generated.v2_all import (
+ InputImageContentItem,
+ InputImageFunctionCallOutputContentItem,
+ )
+
+ assert InputImageContentItem.__name__ == "InputImageContentItem"
+ assert (
+ InputImageFunctionCallOutputContentItem.__name__
+ == "InputImageFunctionCallOutputContentItem"
+ )
+
+
def test_runtime_package_template_has_no_checked_in_binaries() -> None:
runtime_root = ROOT.parent / "python-runtime" / "src" / "codex_cli_bin"
assert sorted(