Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Support image references by file ID in inputs and tool outputs (#45794)

Krish Chainani7b8b17b97a5f
diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs
index 0ac74e9b8c..a576251e5a 100644
--- a/codex-rs/analytics/src/analytics_client_tests.rs
+++ b/codex-rs/analytics/src/analytics_client_tests.rs
@@ -126,6 +126,7 @@ use codex_app_server_protocol::GuardianApprovalReviewAction;
use codex_app_server_protocol::GuardianApprovalReviewStatus;
use codex_app_server_protocol::GuardianCommandSource as AppServerGuardianCommandSource;
use codex_app_server_protocol::ImageGenerationItem;
+use codex_app_server_protocol::ImageReference;
use codex_app_server_protocol::InitializeCapabilities;
use codex_app_server_protocol::InitializeParams;
use codex_app_server_protocol::ItemCompletedNotification;
@@ -396,7 +397,9 @@ fn sample_turn_start_request(thread_id: &str, request_id: i64) -> ClientRequest
text_elements: vec![],
},
UserInput::Image {
- url: "https://example.com/a.png".to_string(),
+ image: ImageReference::Inline {
+ url: "https://example.com/a.png".to_string(),
+ },
detail: None,
},
],
diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json
index d05de8638d..ac25bc7822 100644
--- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json
+++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json
@@ -672,6 +672,18 @@
],
"title": "ImageUrlContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdContentItem",
+ "type": "object"
}
],
"properties": {
@@ -1442,6 +1454,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -5663,6 +5687,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -5681,14 +5731,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json
index 6d7bc110d5..b4f02ff44c 100644
--- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json
+++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json
@@ -1771,6 +1771,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -6469,6 +6481,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -6487,14 +6525,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json
index 2863a575a9..fa0ba67749 100644
--- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json
+++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json
@@ -10484,6 +10484,18 @@
],
"title": "ImageUrlContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdContentItem",
+ "type": "object"
}
],
"properties": {
@@ -12355,6 +12367,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -25427,6 +25451,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -25445,14 +25495,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json
index c0787c3bfb..3b8f1e185f 100644
--- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json
+++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json
@@ -6364,6 +6364,18 @@
],
"title": "ImageUrlContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdContentItem",
+ "type": "object"
}
],
"properties": {
@@ -8235,6 +8247,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -23141,6 +23165,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -23159,14 +23209,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json
index dc2d0dc619..b0757fd2e4 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json
@@ -371,6 +371,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -1614,6 +1626,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -1632,14 +1670,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json
index c153e128b3..6018fa212f 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json
@@ -371,6 +371,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -1614,6 +1626,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -1632,14 +1670,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json
index c05b788d8a..e9c7bf6e43 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json
@@ -92,6 +92,18 @@
],
"title": "ImageUrlContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdContentItem",
+ "type": "object"
}
],
"properties": {
@@ -210,6 +222,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json
index cddc22f59f..a058b676e6 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json
@@ -511,6 +511,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -1944,6 +1956,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -1962,14 +2000,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json
index a10d95cf00..dc0f83ba14 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json
@@ -592,6 +592,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -2684,6 +2696,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -2702,14 +2740,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json
index f467944e67..f8427717dc 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json
@@ -371,6 +371,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -1630,6 +1642,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -1648,14 +1686,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json
index 537af065c3..9f4be687e0 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json
@@ -514,6 +514,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -2475,6 +2487,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -2493,14 +2531,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json
index c76c1d2491..32cfc6fe96 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json
@@ -514,6 +514,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -2475,6 +2487,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -2493,14 +2531,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json
index 510c26c2ee..a04d78dc33 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json
@@ -514,6 +514,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -2475,6 +2487,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -2493,14 +2531,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json
index 753f062836..2d8a10968f 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json
@@ -154,6 +154,18 @@
],
"title": "ImageUrlContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdContentItem",
+ "type": "object"
}
],
"properties": {
@@ -272,6 +284,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json
index 3a6fb8bdaa..cb8c0052c3 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json
@@ -608,6 +608,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -2762,6 +2774,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -2780,14 +2818,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json
index 8d8a69f51d..2b57af18c2 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json
@@ -514,6 +514,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -2475,6 +2487,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -2493,14 +2531,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json
index da04bf0a27..9e0f4155c2 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json
@@ -592,6 +592,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -2684,6 +2696,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -2702,14 +2740,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json
index 084cc6522d..6832e8318e 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json
@@ -514,6 +514,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -2475,6 +2487,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -2493,14 +2531,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json
index 3e06a271fa..59ce4d96ba 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json
@@ -511,6 +511,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -1944,6 +1956,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -1962,14 +2000,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json
index d396062af1..f2c1ce42c9 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json
@@ -514,6 +514,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -2475,6 +2487,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -2493,14 +2531,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json
index 9ca8c3b087..2e24d09644 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json
@@ -511,6 +511,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -1944,6 +1956,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -1962,14 +2000,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json
index 68f4b3dff9..5b9105d8e9 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json
@@ -178,6 +178,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -559,6 +571,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -577,14 +615,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json
index f2d2251e03..c715c11287 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json
@@ -511,6 +511,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -1944,6 +1956,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -1962,14 +2000,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json
index bdf800c52a..0781a4f439 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json
@@ -511,6 +511,18 @@
],
"title": "ImageUrlFunctionCallOutputContentItem",
"type": "object"
+ },
+ {
+ "properties": {
+ "file_id": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "file_id"
+ ],
+ "title": "FileIdFunctionCallOutputContentItem",
+ "type": "object"
}
],
"properties": {
@@ -1944,6 +1956,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -1962,14 +2000,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json
index 2a3627cbd0..7dbdefa046 100644
--- a/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json
+++ b/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json
@@ -105,6 +105,32 @@
"type": "object"
},
{
+ "anyOf": [
+ {
+ "properties": {
+ "url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "url"
+ ],
+ "title": "UrlUserInput",
+ "type": "object"
+ },
+ {
+ "properties": {
+ "fileId": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "fileId"
+ ],
+ "title": "FileIdUserInput",
+ "type": "object"
+ }
+ ],
"properties": {
"detail": {
"anyOf": [
@@ -123,14 +149,10 @@
],
"title": "ImageUserInputType",
"type": "string"
- },
- "url": {
- "type": "string"
}
},
"required": [
- "type",
- "url"
+ "type"
],
"title": "ImageUserInput",
"type": "object"
diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst
index 7b2430db77..96e4c83090 100644
Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst differ
diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst
index e720fc5288..b339e9b4ab 100644
Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst differ
diff --git a/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts b/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts
index dd5b0a1929..0ad09f18e4 100644
--- a/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts
+++ b/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts
@@ -3,4 +3,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { ImageDetail } from "./ImageDetail";
-export type ContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "output_text", text: string, };
+export type ContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, } | { file_id: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "output_text", text: string, };
diff --git a/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts b/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts
index 8ebefc596e..5fa3d63afe 100644
--- a/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts
+++ b/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts
@@ -7,4 +7,4 @@ import type { ImageDetail } from "./ImageDetail";
* Responses API compatible content items that can be returned by a tool call.
* This is a subset of ContentItem with the types we support as function call outputs.
*/
-export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "encrypted_content", encrypted_content: string, };
+export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, } | { file_id: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "encrypted_content", encrypted_content: string, };
diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/UserInput.ts b/codex-rs/app-server-protocol/schema/typescript/v2/UserInput.ts
index c268cb4f8d..390d9dd59c 100644
--- a/codex-rs/app-server-protocol/schema/typescript/v2/UserInput.ts
+++ b/codex-rs/app-server-protocol/schema/typescript/v2/UserInput.ts
@@ -8,4 +8,4 @@ export type UserInput = { "type": "text", text: string,
/**
* UI-defined spans within `text` used to render or persist special elements.
*/
-text_elements: Array<TextElement>, } | { "type": "image", detail?: ImageDetail, url: string, } | { "type": "localImage", detail?: ImageDetail, path: string, } | { "type": "audio", url: string, } | { "type": "localAudio", path: string, } | { "type": "skill", name: string, path: string, } | { "type": "mention", name: string, path: string, };
+text_elements: Array<TextElement>, } | { "type": "image", detail?: ImageDetail, } & ({ url: string, } | { fileId: string, }) | { "type": "localImage", detail?: ImageDetail, path: string, } | { "type": "audio", url: string, } | { "type": "localAudio", path: string, } | { "type": "skill", name: string, path: string, } | { "type": "mention", name: string, path: string, };
diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs
index 0c46d8c613..4b6b4168f8 100644
--- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs
+++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs
@@ -11,6 +11,7 @@ use crate::protocol::v2::CollabAgentToolCallStatus;
use crate::protocol::v2::CommandExecutionStatus;
use crate::protocol::v2::DynamicToolCallOutputContentItem;
use crate::protocol::v2::DynamicToolCallStatus;
+use crate::protocol::v2::ImageReference;
use crate::protocol::v2::McpToolCallAppContext;
use crate::protocol::v2::McpToolCallError;
use crate::protocol::v2::McpToolCallResult;
@@ -54,6 +55,7 @@ use codex_protocol::protocol::TurnAbortedEvent;
use codex_protocol::protocol::TurnCompleteEvent;
use codex_protocol::protocol::TurnStartedEvent;
use codex_protocol::protocol::UserMessageEvent;
+use codex_protocol::protocol::UserMessageImageKind;
use codex_protocol::protocol::ViewImageToolCallEvent;
use codex_protocol::protocol::WebSearchBeginEvent;
use codex_protocol::protocol::WebSearchEndEvent;
@@ -1511,14 +1513,62 @@ impl ThreadHistoryBuilder {
.collect(),
});
}
- if let Some(images) = &payload.images {
+ let has_complete_image_order = payload.has_complete_image_order();
+ if has_complete_image_order {
+ let mut inline_index = 0;
+ let mut file_index = 0;
+ for image_kind in &payload.image_order {
+ let (image, detail) = match image_kind {
+ UserMessageImageKind::Inline => {
+ let Some(image) = payload
+ .images
+ .as_deref()
+ .and_then(|images| images.get(inline_index))
+ else {
+ continue;
+ };
+ let detail = payload.image_details.get(inline_index).copied().flatten();
+ inline_index += 1;
+ (ImageReference::Inline { url: image.clone() }, detail)
+ }
+ UserMessageImageKind::File => {
+ let Some(file_id) = payload
+ .file_ids
+ .as_deref()
+ .and_then(|file_ids| file_ids.get(file_index))
+ else {
+ continue;
+ };
+ let detail = payload.file_id_details.get(file_index).copied().flatten();
+ file_index += 1;
+ (
+ ImageReference::File {
+ file_id: file_id.clone(),
+ },
+ detail,
+ )
+ }
+ };
+ content.push(UserInput::Image { image, detail });
+ }
+ } else if let Some(images) = &payload.images {
for (idx, image) in images.iter().enumerate() {
content.push(UserInput::Image {
- url: image.clone(),
+ image: ImageReference::Inline { url: image.clone() },
detail: payload.image_details.get(idx).copied().flatten(),
});
}
}
+ if !has_complete_image_order && let Some(file_ids) = &payload.file_ids {
+ for (idx, file_id) in file_ids.iter().enumerate() {
+ content.push(UserInput::Image {
+ image: ImageReference::File {
+ file_id: file_id.clone(),
+ },
+ detail: payload.file_id_details.get(idx).copied().flatten(),
+ });
+ }
+ }
for (idx, path) in payload.local_images.iter().enumerate() {
content.push(UserInput::LocalImage {
path: path.clone(),
@@ -1824,7 +1874,9 @@ mod tests {
text_elements: Vec::new(),
},
UserInput::Image {
- url: "https://example.com/one.png".into(),
+ image: ImageReference::Inline {
+ url: "https://example.com/one.png".into(),
+ },
detail: None,
}
],
@@ -1999,6 +2051,9 @@ mod tests {
message: "inspect these".into(),
images: Some(vec!["https://example.com/image.png".into()]),
image_details: vec![Some(ImageDetail::Original)],
+ file_ids: Some(vec!["file_123".into()]),
+ file_id_details: vec![Some(ImageDetail::High)],
+ image_order: vec![UserMessageImageKind::File, UserMessageImageKind::Inline],
local_images: vec![local_image_path.clone()],
local_image_details: vec![Some(ImageDetail::Original)],
audio: Some(vec!["https://example.com/audio.mp3".into()]),
@@ -2021,7 +2076,15 @@ mod tests {
text_elements: Vec::new(),
},
UserInput::Image {
- url: "https://example.com/image.png".into(),
+ image: ImageReference::File {
+ file_id: "file_123".into(),
+ },
+ detail: Some(ImageDetail::High),
+ },
+ UserInput::Image {
+ image: ImageReference::Inline {
+ url: "https://example.com/image.png".into(),
+ },
detail: Some(ImageDetail::Original),
},
UserInput::LocalImage {
@@ -2039,6 +2102,43 @@ mod tests {
);
}
+ /// Incomplete ordering metadata must not drop references from the legacy split arrays.
+ #[test]
+ fn incomplete_image_order_falls_back_to_legacy_order() {
+ let events = vec![RolloutItem::EventMsg(EventMsg::UserMessage(
+ UserMessageEvent {
+ images: Some(vec!["https://example.com/image.png".into()]),
+ file_ids: Some(vec!["file_123".into()]),
+ image_order: vec![UserMessageImageKind::File],
+ ..Default::default()
+ },
+ ))];
+
+ let turns = build_turns_from_rollout_items(&events);
+
+ assert_eq!(
+ turns[0].items[0],
+ ThreadItem::UserMessage {
+ id: "item-1".into(),
+ client_id: None,
+ content: vec![
+ UserInput::Image {
+ image: ImageReference::Inline {
+ url: "https://example.com/image.png".into(),
+ },
+ detail: None,
+ },
+ UserInput::Image {
+ image: ImageReference::File {
+ file_id: "file_123".into(),
+ },
+ detail: None,
+ },
+ ],
+ }
+ );
+ }
+
#[test]
fn ignores_user_message_item_lifecycle_events() {
let turn_id = "turn-1";
diff --git a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs
index 496116f4e5..29cac91c98 100644
--- a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs
+++ b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs
@@ -29,6 +29,7 @@ use codex_protocol::models::AdditionalPermissionProfile as CoreAdditionalPermiss
use codex_protocol::models::BUILT_IN_PERMISSION_PROFILE_WORKSPACE;
use codex_protocol::models::FileSystemPermissions as CoreFileSystemPermissions;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference as CoreImageReference;
use codex_protocol::models::MessagePhase;
use codex_protocol::models::NetworkPermissions as CoreNetworkPermissions;
use codex_protocol::models::WebSearchAction as CoreWebSearchAction;
@@ -3037,7 +3038,9 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
text_elements: Vec::new(),
},
CoreUserInput::Image {
- image_url: "https://example.com/image.png".to_string(),
+ image: CoreImageReference::Inline {
+ image_url: "https://example.com/image.png".to_string(),
+ },
detail: Some(ImageDetail::Original),
},
CoreUserInput::LocalImage {
@@ -3072,7 +3075,9 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
text_elements: Vec::new(),
},
UserInput::Image {
- url: "https://example.com/image.png".to_string(),
+ image: ImageReference::Inline {
+ url: "https://example.com/image.png".to_string(),
+ },
detail: Some(ImageDetail::Original),
},
UserInput::LocalImage {
@@ -3609,16 +3614,84 @@ fn mcp_tool_call_app_context_serializes_missing_mixed_version_fields_as_null() {
);
}
+/// Keeps the existing URL form stable while exposing file-backed images as `fileId`.
+#[test]
+fn user_input_image_references_round_trip_with_stable_wire_shapes() {
+ let cases = [
+ (
+ UserInput::Image {
+ image: ImageReference::Inline {
+ url: "data:image/png;base64,AAA".to_string(),
+ },
+ detail: Some(ImageDetail::High),
+ },
+ json!({
+ "type": "image",
+ "url": "data:image/png;base64,AAA",
+ "detail": "high",
+ }),
+ ),
+ (
+ UserInput::Image {
+ image: ImageReference::File {
+ file_id: "file_123".to_string(),
+ },
+ detail: Some(ImageDetail::Original),
+ },
+ json!({
+ "type": "image",
+ "fileId": "file_123",
+ "detail": "original",
+ }),
+ ),
+ ];
+
+ for (input, wire_value) in cases {
+ assert_eq!(
+ serde_json::to_value(&input).expect("user input should serialize"),
+ wire_value,
+ );
+ assert_eq!(
+ serde_json::from_value::<UserInput>(wire_value).expect("user input should deserialize"),
+ input,
+ );
+ }
+}
+
+/// Preserves durable file identity in both directions at the Core boundary.
+#[test]
+fn file_image_user_input_converts_both_directions() {
+ let app_server_input = UserInput::Image {
+ image: ImageReference::File {
+ file_id: "file_123".to_string(),
+ },
+ detail: Some(ImageDetail::High),
+ };
+ let core_input = CoreUserInput::Image {
+ image: CoreImageReference::File {
+ file_id: "file_123".to_string(),
+ },
+ detail: Some(ImageDetail::High),
+ };
+
+ assert_eq!(app_server_input.clone().into_core(), core_input);
+ assert_eq!(UserInput::from(core_input), app_server_input);
+}
+
#[test]
fn user_input_into_core_preserves_media_fields() {
assert_eq!(
UserInput::Image {
- url: "https://example.com/image.png".to_string(),
+ image: ImageReference::Inline {
+ url: "https://example.com/image.png".to_string(),
+ },
detail: Some(ImageDetail::Original),
}
.into_core(),
CoreUserInput::Image {
- image_url: "https://example.com/image.png".to_string(),
+ image: CoreImageReference::Inline {
+ image_url: "https://example.com/image.png".to_string(),
+ },
detail: Some(ImageDetail::Original),
}
);
diff --git a/codex-rs/app-server-protocol/src/protocol/v2/turn.rs b/codex-rs/app-server-protocol/src/protocol/v2/turn.rs
index a219c5f660..3435ae6402 100644
--- a/codex-rs/app-server-protocol/src/protocol/v2/turn.rs
+++ b/codex-rs/app-server-protocol/src/protocol/v2/turn.rs
@@ -11,6 +11,7 @@ use codex_protocol::config_types::Personality;
use codex_protocol::config_types::ReasoningSummary;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference as CoreImageReference;
use codex_protocol::openai_models::ReasoningEffort;
use codex_protocol::plan_tool::PlanItemArg as CorePlanItemArg;
use codex_protocol::plan_tool::StepStatus as CorePlanStepStatus;
@@ -401,6 +402,21 @@ impl From<TextElement> for CoreTextElement {
}
}
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
+#[serde(untagged)]
+#[ts(untagged)]
+#[ts(export_to = "v2/")]
+pub enum ImageReference {
+ Inline {
+ url: String,
+ },
+ File {
+ #[serde(rename = "fileId")]
+ #[ts(rename = "fileId")]
+ file_id: String,
+ },
+}
+
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(tag = "type", rename_all = "camelCase")]
#[ts(tag = "type")]
@@ -413,10 +429,11 @@ pub enum UserInput {
text_elements: Vec<TextElement>,
},
Image {
+ #[serde(flatten)]
+ image: ImageReference,
#[serde(default)]
#[ts(optional)]
detail: Option<ImageDetail>,
- url: String,
},
LocalImage {
#[serde(default)]
@@ -450,8 +467,11 @@ impl UserInput {
text,
text_elements: text_elements.into_iter().map(Into::into).collect(),
},
- UserInput::Image { url, detail } => CoreUserInput::Image {
- image_url: url,
+ UserInput::Image { image, detail } => CoreUserInput::Image {
+ image: match image {
+ ImageReference::Inline { url } => CoreImageReference::Inline { image_url: url },
+ ImageReference::File { file_id } => CoreImageReference::File { file_id },
+ },
detail,
},
UserInput::LocalImage { path, detail } => CoreUserInput::LocalImage { path, detail },
@@ -473,8 +493,13 @@ impl From<CoreUserInput> for UserInput {
text,
text_elements: text_elements.into_iter().map(Into::into).collect(),
},
- CoreUserInput::Image { image_url, detail } => UserInput::Image {
- url: image_url,
+ CoreUserInput::Image { image, detail } => UserInput::Image {
+ image: match image {
+ CoreImageReference::Inline { image_url } => {
+ ImageReference::Inline { url: image_url }
+ }
+ CoreImageReference::File { file_id } => ImageReference::File { file_id },
+ },
detail,
},
CoreUserInput::LocalImage { path, detail } => UserInput::LocalImage { path, detail },
diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs
index a051c867db..1de7f03f1b 100644
--- a/codex-rs/app-server/src/request_processors/turn_processor.rs
+++ b/codex-rs/app-server/src/request_processors/turn_processor.rs
@@ -3,12 +3,13 @@ use super::*;
use codex_agent_extension::AgentInvocation;
use codex_agent_extension::AgentRun;
use codex_agent_extension::AgentRunner;
+use codex_app_server_protocol::ImageReference as V2ImageReference;
use codex_protocol::error::CodexErrorDetails;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
-use codex_protocol::models::ImageReference;
+use codex_protocol::models::ImageReference as CoreImageReference;
use codex_protocol::protocol::AdditionalContextEntry as CoreAdditionalContextEntry;
use codex_protocol::protocol::AdditionalContextKind as CoreAdditionalContextKind;
use codex_protocol::protocol::TurnSettingsUpdate;
@@ -24,7 +25,10 @@ pub(super) fn validate_user_input_image_urls(
if input.iter().any(|item| {
matches!(
item,
- V2UserInput::Image { url, .. } if is_remote_image_url(url)
+ V2UserInput::Image {
+ image: V2ImageReference::Inline { url },
+ ..
+ } if is_remote_image_url(url)
)
}) {
return Err(invalid_request(REMOTE_IMAGE_URL_ERROR));
@@ -37,7 +41,7 @@ fn validate_response_item_image_urls(items: &[ResponseItem]) -> Result<(), JSONR
ResponseItem::Message { content, .. } => content.iter().any(|item| {
matches!(
item,
- ContentItem::InputImage { image: ImageReference::Inline { image_url }, .. } if is_remote_image_url(image_url)
+ ContentItem::InputImage { image: CoreImageReference::Inline { image_url }, .. } if is_remote_image_url(image_url)
)
}),
ResponseItem::FunctionCallOutput { output, .. }
@@ -46,7 +50,7 @@ fn validate_response_item_image_urls(items: &[ResponseItem]) -> Result<(), JSONR
content.iter().any(|item| {
matches!(
item,
- FunctionCallOutputContentItem::InputImage { image: ImageReference::Inline { image_url }, .. }
+ FunctionCallOutputContentItem::InputImage { image: CoreImageReference::Inline { image_url }, .. }
if is_remote_image_url(image_url)
)
})
diff --git a/codex-rs/app-server/tests/suite/v2/guardian_v2.rs b/codex-rs/app-server/tests/suite/v2/guardian_v2.rs
index bc2d3685ac..90d07c6a50 100644
--- a/codex-rs/app-server/tests/suite/v2/guardian_v2.rs
+++ b/codex-rs/app-server/tests/suite/v2/guardian_v2.rs
@@ -25,6 +25,7 @@ use axum::routing::post;
use codex_app_server_protocol::ApprovalsReviewer;
use codex_app_server_protocol::AskForApproval;
use codex_app_server_protocol::ClientRequest;
+use codex_app_server_protocol::ImageReference;
use codex_app_server_protocol::ItemCompletedNotification;
use codex_app_server_protocol::ItemGuardianApprovalReviewStartedNotification;
use codex_app_server_protocol::McpServerElicitationRequest;
@@ -1031,7 +1032,9 @@ async fn guardian_v2_routes_scoped_tool_approvals(
}
if mixed_evidence {
turn_input.push(UserInput::Image {
- url: EVIDENCE_IMAGE.to_owned(),
+ image: ImageReference::Inline {
+ url: EVIDENCE_IMAGE.to_owned(),
+ },
detail: None,
});
}
diff --git a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs
index b5bae46cd2..e92bb49e8e 100644
--- a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs
+++ b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs
@@ -9,6 +9,7 @@ use app_test_support::TestAppServer;
use app_test_support::write_chatgpt_auth;
use codex_app_server_protocol::ImageGenerationFailure;
use codex_app_server_protocol::ImageGenerationItem;
+use codex_app_server_protocol::ImageReference as V2ImageReference;
use codex_app_server_protocol::ItemCompletedNotification;
use codex_app_server_protocol::ThreadItem;
use codex_app_server_protocol::ThreadReadParams;
@@ -657,7 +658,9 @@ async fn transparent_image_edit_preserves_metadata_and_recent_pathless_image() -
text_elements: Vec::new(),
},
V2UserInput::Image {
- url: image_url.to_string(),
+ image: V2ImageReference::Inline {
+ url: image_url.to_string(),
+ },
detail: None,
},
],
diff --git a/codex-rs/app-server/tests/suite/v2/thread_read.rs b/codex-rs/app-server/tests/suite/v2/thread_read.rs
index 85c7a1e86a..860f150869 100644
--- a/codex-rs/app-server/tests/suite/v2/thread_read.rs
+++ b/codex-rs/app-server/tests/suite/v2/thread_read.rs
@@ -12,6 +12,7 @@ use codex_app_server::in_process::InProcessStartArgs;
use codex_app_server_protocol::ClientInfo;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::DeprecationNoticeNotification;
+use codex_app_server_protocol::ImageReference;
use codex_app_server_protocol::InitializeCapabilities;
use codex_app_server_protocol::InitializeParams;
use codex_app_server_protocol::JSONRPCError;
@@ -234,6 +235,80 @@ async fn thread_read_can_include_turns() -> Result<()> {
Ok(())
}
+/// Preserves a file-backed image across a completed turn, including a cold history read.
+#[tokio::test]
+async fn thread_read_preserves_file_id_from_completed_turn() -> Result<()> {
+ let server = create_mock_responses_server_repeating_assistant("Done").await;
+ let codex_home = TempDir::new()?;
+ MockResponsesConfig::new(&server.uri()).write(codex_home.path())?;
+
+ let mut mcp = TestAppServer::builder()
+ .with_codex_home(codex_home.path())
+ .build_initialized()
+ .await?;
+ let start_id = mcp
+ .send_thread_start_request_with_auto_env(ThreadStartParams {
+ model: Some("mock-model".to_string()),
+ ..Default::default()
+ })
+ .await?;
+ let ThreadStartResponse { thread, .. } =
+ timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(start_id)).await??;
+ let image = UserInput::Image {
+ image: ImageReference::File {
+ file_id: "file_123".to_string(),
+ },
+ detail: None,
+ };
+
+ timeout(
+ DEFAULT_READ_TIMEOUT,
+ mcp.start_turn_and_wait_for_completion(TurnStartParams {
+ thread_id: thread.id.clone(),
+ input: vec![image.clone()],
+ ..Default::default()
+ }),
+ )
+ .await??;
+
+ let read_id = mcp
+ .send_thread_read_request(ThreadReadParams {
+ thread_id: thread.id.clone(),
+ include_turns: true,
+ })
+ .await?;
+ let ThreadReadResponse {
+ thread: loaded_thread,
+ } = timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(read_id)).await??;
+ let ThreadItem::UserMessage { content, .. } = &loaded_thread.turns[0].items[0] else {
+ panic!("expected completed turn to start with a user message");
+ };
+ assert_eq!(content, &vec![image.clone()]);
+
+ timeout(DEFAULT_READ_TIMEOUT, mcp.shutdown_gracefully()).await??;
+ drop(mcp);
+
+ let mut mcp = TestAppServer::builder()
+ .with_codex_home(codex_home.path())
+ .build_initialized()
+ .await?;
+ let read_id = mcp
+ .send_thread_read_request(ThreadReadParams {
+ thread_id: thread.id,
+ include_turns: true,
+ })
+ .await?;
+ let ThreadReadResponse {
+ thread: restarted_thread,
+ } = timeout(DEFAULT_READ_TIMEOUT, mcp.read_response(read_id)).await??;
+ let ThreadItem::UserMessage { content, .. } = &restarted_thread.turns[0].items[0] else {
+ panic!("expected persisted turn to start with a user message");
+ };
+ assert_eq!(content, &vec![image]);
+
+ Ok(())
+}
+
#[tokio::test]
async fn paginated_stored_thread_routes_projected_turns() -> Result<()> {
let server = create_mock_responses_server_repeating_assistant("Done").await;
diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs
index 4bb6eafcfb..311b59cdae 100644
--- a/codex-rs/app-server/tests/suite/v2/turn_start.rs
+++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs
@@ -34,6 +34,7 @@ use codex_app_server_protocol::CyberAccessProgram;
use codex_app_server_protocol::FileChangeApprovalDecision;
use codex_app_server_protocol::FileChangePatchUpdatedNotification;
use codex_app_server_protocol::FileChangeRequestApprovalResponse;
+use codex_app_server_protocol::ImageReference as V2ImageReference;
use codex_app_server_protocol::ItemCompletedNotification;
use codex_app_server_protocol::ItemStartedNotification;
use codex_app_server_protocol::JSONRPCError;
@@ -267,7 +268,15 @@ async fn turn_start_omits_notification_media_without_changing_model_input() -> R
text_elements: Vec::new(),
},
V2UserInput::Image {
- url: TINY_PNG_DATA_URL.to_string(),
+ image: V2ImageReference::Inline {
+ url: TINY_PNG_DATA_URL.to_string(),
+ },
+ detail: None,
+ },
+ V2UserInput::Image {
+ image: V2ImageReference::File {
+ file_id: "file_123".to_string(),
+ },
detail: None,
},
],
@@ -315,6 +324,8 @@ async fn turn_start_omits_notification_media_without_changing_model_input() -> R
content[0]["type"].as_str(),
Some("text" | "input_text")
));
+ assert!(content[0].get("fileId").is_none());
+ assert!(content[0].get("file_id").is_none());
user_message_notifications.push(notification.method);
}
}
@@ -332,8 +343,17 @@ async fn turn_start_omits_notification_media_without_changing_model_input() -> R
);
let model_input_images = received_response_input_images(&server).await?;
- assert_eq!(model_input_images.len(), 1);
- assert_eq!(model_input_images[0]["image_url"], TINY_PNG_DATA_URL);
+ assert_eq!(model_input_images.len(), 2);
+ assert!(
+ model_input_images
+ .iter()
+ .any(|image| image["image_url"] == TINY_PNG_DATA_URL)
+ );
+ assert!(
+ model_input_images
+ .iter()
+ .any(|image| image["file_id"] == "file_123")
+ );
Ok(())
}
@@ -1393,7 +1413,9 @@ async fn turn_start_tracks_thread_originator_in_analytics() -> Result<()> {
thread_id: thread.id.clone(),
client_user_message_id: None,
input: vec![V2UserInput::Image {
- url: TINY_PNG_DATA_URL.to_string(),
+ image: V2ImageReference::Inline {
+ url: TINY_PNG_DATA_URL.to_string(),
+ },
detail: None,
}],
turn_trigger: Some("user".to_string()),
diff --git a/codex-rs/core/src/compact_remote_v2_images.rs b/codex-rs/core/src/compact_remote_v2_images.rs
index 0575e07f01..94cbdc239e 100644
--- a/codex-rs/core/src/compact_remote_v2_images.rs
+++ b/codex-rs/core/src/compact_remote_v2_images.rs
@@ -25,6 +25,10 @@ pub(super) fn content_item_token_count(item: &ContentItem) -> usize {
image_url, *detail,
)))
.unwrap_or(usize::MAX),
+ ContentItem::InputImage {
+ image: ImageReference::File { .. },
+ ..
+ } => 0,
ContentItem::InputAudio { .. } => 0,
}
}
diff --git a/codex-rs/core/src/context/node_repl_review_evidence.rs b/codex-rs/core/src/context/node_repl_review_evidence.rs
index cd49329a13..d5bad1ea45 100644
--- a/codex-rs/core/src/context/node_repl_review_evidence.rs
+++ b/codex-rs/core/src/context/node_repl_review_evidence.rs
@@ -63,7 +63,14 @@ impl NodeReplReviewResponse {
.saturating_add(self.items.iter().fold(0_usize, |bytes, item| {
bytes.saturating_add(match item {
Text { text, .. } => text.len(),
- Image { image_url, .. } => image_url.len(),
+ Image {
+ image: ImageReference::Inline { image_url },
+ ..
+ } => image_url.len(),
+ Image {
+ image: ImageReference::File { file_id },
+ ..
+ } => file_id.len(),
_ => 0,
})
}))
@@ -116,14 +123,15 @@ impl NodeReplReviewEvidence {
.iter()
.flat_map(|response| &response.items)
.filter_map(|item| match item {
- Image { image_url, detail } if seen_images.insert(image_url) => {
- Some(ContentItem::InputImage {
- image: ImageReference::Inline {
- image_url: image_url.clone(),
- },
- detail: *detail,
- })
- }
+ Image {
+ image: ImageReference::Inline { image_url },
+ detail,
+ } if seen_images.insert(image_url) => Some(ContentItem::InputImage {
+ image: ImageReference::Inline {
+ image_url: image_url.clone(),
+ },
+ detail: *detail,
+ }),
_ => None,
})
.collect()
diff --git a/codex-rs/core/src/context/node_repl_review_evidence_tests.rs b/codex-rs/core/src/context/node_repl_review_evidence_tests.rs
index 290aa7d31a..2df586ec34 100644
--- a/codex-rs/core/src/context/node_repl_review_evidence_tests.rs
+++ b/codex-rs/core/src/context/node_repl_review_evidence_tests.rs
@@ -1,3 +1,4 @@
+use codex_protocol::models::ImageReference;
use codex_protocol::user_input::UserInput;
use codex_utils_output_truncation::approx_bytes_for_tokens;
use pretty_assertions::assert_eq;
@@ -15,7 +16,18 @@ fn text_input(text: &str) -> UserInput {
fn image_input(image_url: &str) -> UserInput {
UserInput::Image {
- image_url: image_url.to_string(),
+ image: ImageReference::Inline {
+ image_url: image_url.to_string(),
+ },
+ detail: None,
+ }
+}
+
+fn file_image_input(file_id: &str) -> UserInput {
+ UserInput::Image {
+ image: ImageReference::File {
+ file_id: file_id.to_string(),
+ },
detail: None,
}
}
@@ -121,6 +133,36 @@ fn evidence_preserves_tail_text_after_oversized_image_response() {
assert!(rendered_text(&inputs).contains("FINAL IMPORTANT"));
}
+#[test]
+fn file_backed_images_are_retained_but_omitted_from_guardian_views() {
+ let evidence = NodeReplReviewEvidence::default();
+ let file_id = format!("file_{}", "x".repeat(4_096));
+ let items = vec![text_input("visible"), file_image_input(&file_id)];
+
+ evidence.record("browser", "cell", "file", items.clone());
+
+ let snapshot = evidence.snapshot_since(/*reviewed_sequence*/ 0).unwrap();
+ assert_eq!(snapshot.responses.len(), 1);
+ assert_eq!(snapshot.responses[0].items, items);
+ assert!(snapshot.responses[0].has_images());
+ assert!(snapshot.responses[0].retained_bytes() >= file_id.len());
+ assert_eq!(evidence.images(), Vec::new());
+
+ let rendered = snapshot
+ .context(NodeReplReviewEvidenceMode::Multimodal)
+ .render_inputs();
+ assert!(rendered_text(&rendered).contains("visible"));
+ assert!(!rendered.iter().any(|item| {
+ matches!(
+ item,
+ UserInput::Image {
+ image: ImageReference::File { .. },
+ ..
+ }
+ )
+ }));
+}
+
#[test]
fn evidence_evicts_complete_oldest_responses_and_rejects_oversized_items() {
let evidence = NodeReplReviewEvidence::default();
diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs
index 69e4e9787c..944efc2929 100644
--- a/codex-rs/core/src/context_manager/history.rs
+++ b/codex-rs/core/src/context_manager/history.rs
@@ -930,6 +930,7 @@ static ORIGINAL_IMAGE_ESTIMATE_CACHE: LazyLock<BlockingLruCache<[u8; 20], Option
});
fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
+ // TODO(kc) Account for file-backed image size after its token-cost contract is defined.
match item {
ResponseItem::Message { content, .. } => content
.iter()
@@ -941,6 +942,10 @@ fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
image: ImageReference::Inline { image_url },
detail,
} => estimate_image_bytes(image_url, *detail),
+ ContentItem::InputImage {
+ image: ImageReference::File { .. },
+ ..
+ } => 0,
ContentItem::InputAudio { audio_url } => estimate_audio_bytes(audio_url),
})
.fold(0i64, i64::saturating_add),
@@ -1147,6 +1152,10 @@ fn estimate_function_output_bytes(output: &FunctionCallOutputBody) -> i64 {
image: ImageReference::Inline { image_url },
detail,
} => estimate_image_bytes(image_url, *detail),
+ FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::File { .. },
+ ..
+ } => 0,
FunctionCallOutputContentItem::InputAudio { audio_url } => {
estimate_audio_bytes(audio_url)
}
diff --git a/codex-rs/core/src/event_mapping.rs b/codex-rs/core/src/event_mapping.rs
index ed983fe644..176e33da17 100644
--- a/codex-rs/core/src/event_mapping.rs
+++ b/codex-rs/core/src/event_mapping.rs
@@ -5,7 +5,6 @@ use codex_protocol::items::TurnItem;
use codex_protocol::items::UserMessageItem;
use codex_protocol::items::WebSearchItem;
use codex_protocol::models::ContentItem;
-use codex_protocol::models::ImageReference;
use codex_protocol::models::MessagePhase;
use codex_protocol::models::ReasoningItemContent;
use codex_protocol::models::ReasoningItemReasoningSummary;
@@ -127,12 +126,9 @@ fn parse_user_message(message: &[ContentItem]) -> Option<UserMessageItem> {
text_elements: Vec::new(),
});
}
- ContentItem::InputImage {
- image: ImageReference::Inline { image_url },
- detail,
- } => {
+ ContentItem::InputImage { image, detail } => {
content.push(UserInput::Image {
- image_url: image_url.clone(),
+ image: image.clone(),
detail: *detail,
});
}
diff --git a/codex-rs/core/src/event_mapping_tests.rs b/codex-rs/core/src/event_mapping_tests.rs
index b6ac81c9da..d6d9f59f2c 100644
--- a/codex-rs/core/src/event_mapping_tests.rs
+++ b/codex-rs/core/src/event_mapping_tests.rs
@@ -110,11 +110,11 @@ fn parses_user_message_with_text_and_two_images() {
text_elements: Vec::new(),
},
UserInput::Image {
- image_url: img1,
+ image: ImageReference::Inline { image_url: img1 },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
UserInput::Image {
- image_url: img2,
+ image: ImageReference::Inline { image_url: img2 },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
];
@@ -124,6 +124,49 @@ fn parses_user_message_with_text_and_two_images() {
}
}
+/// Canonical user-message events must retain opaque file IDs for durable thread history.
+#[test]
+fn parses_user_message_with_file_image() {
+ let item = ResponseItem::Message {
+ id: None,
+ role: "user".to_string(),
+ content: vec![
+ ContentItem::InputImage {
+ image: ImageReference::File {
+ file_id: "file_123".to_string(),
+ },
+ detail: Some(DEFAULT_IMAGE_DETAIL),
+ },
+ ContentItem::InputText {
+ text: "describe it".to_string(),
+ },
+ ],
+ phase: None,
+ internal_chat_message_metadata_passthrough: None,
+ };
+
+ let turn_item = parse_turn_item(&item).expect("expected user message turn item");
+
+ let TurnItem::UserMessage(user) = turn_item else {
+ panic!("expected TurnItem::UserMessage");
+ };
+ assert_eq!(
+ user.content,
+ vec![
+ UserInput::Image {
+ image: ImageReference::File {
+ file_id: "file_123".to_string(),
+ },
+ detail: Some(DEFAULT_IMAGE_DETAIL),
+ },
+ UserInput::Text {
+ text: "describe it".to_string(),
+ text_elements: Vec::new(),
+ },
+ ]
+ );
+}
+
#[test]
fn skips_local_image_label_text() {
let image_url = "data:image/png;base64,abc".to_string();
@@ -158,7 +201,7 @@ fn skips_local_image_label_text() {
TurnItem::UserMessage(user) => {
let expected_content = vec![
UserInput::Image {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
UserInput::Text {
@@ -287,7 +330,7 @@ fn skips_unnamed_image_label_text() {
TurnItem::UserMessage(user) => {
let expected_content = vec![
UserInput::Image {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: Some(DEFAULT_IMAGE_DETAIL),
},
UserInput::Text {
diff --git a/codex-rs/core/src/guardian/review_session.rs b/codex-rs/core/src/guardian/review_session.rs
index 458ed9b970..88cc7acf84 100644
--- a/codex-rs/core/src/guardian/review_session.rs
+++ b/codex-rs/core/src/guardian/review_session.rs
@@ -534,7 +534,9 @@ async fn run_review_on_session(
};
let mut prepared = vec![
ResponseInputItem::from(vec![UserInput::Image {
- image_url: image_url.to_owned(),
+ image: ImageReference::Inline {
+ image_url: image_url.to_owned(),
+ },
detail: *detail,
}])
.into(),
diff --git a/codex-rs/core/src/image_preparation.rs b/codex-rs/core/src/image_preparation.rs
index adc886f7c1..97098705cb 100644
--- a/codex-rs/core/src/image_preparation.rs
+++ b/codex-rs/core/src/image_preparation.rs
@@ -179,6 +179,8 @@ fn prepare_message_content(
metadata: &mut Vec<ImagePreparationMetadata>,
mode: ImagePreparationMode,
) -> Vec<ResizedImage> {
+ // TODO(kc) Prepare file-backed images once Core has an explicit resolution contract. Until
+ // then they pass through while retaining their positions in resize notices.
let image_count = items
.iter()
.filter(|item| matches!(item.content(), ContentItem::InputImage { .. }))
@@ -186,12 +188,14 @@ fn prepare_message_content(
let mut image_number = 0;
let mut resized_images = Vec::new();
for item in items {
+ if matches!(item.content(), ContentItem::InputImage { .. }) {
+ image_number += 1;
+ }
if let ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} = item.content_mut()
{
- image_number += 1;
match prepare_image(image_url, detail, origin, metadata, mode) {
Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => {
resized_images.push(ResizedImage {
@@ -231,12 +235,14 @@ fn prepare_tool_output_content(
let mut image_number = 0;
let mut resized_images = Vec::new();
for item in items {
+ if matches!(item, FunctionCallOutputContentItem::InputImage { .. }) {
+ image_number += 1;
+ }
if let FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
} = item
{
- image_number += 1;
match prepare_image(image_url, detail, origin, metadata, mode) {
Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => {
resized_images.push(ResizedImage {
diff --git a/codex-rs/core/src/image_preparation_tests.rs b/codex-rs/core/src/image_preparation_tests.rs
index a0cfbf5479..ba2324358d 100644
--- a/codex-rs/core/src/image_preparation_tests.rs
+++ b/codex-rs/core/src/image_preparation_tests.rs
@@ -224,8 +224,9 @@ fn preparation_reports_tool_output_item_id() {
);
}
+/// File-backed images retain their sequence positions even though preparation skips them.
#[test]
-fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
+fn resize_notices_count_file_backed_images_and_skip_failed_images() {
let (large_image_url, _) = png_data_url(/*width*/ 2048, /*height*/ 2048);
let (small_image_url, _) = png_data_url(/*width*/ 64, /*height*/ 32);
let mut items = vec![
@@ -233,6 +234,12 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
id: None,
role: "user".to_string(),
content: vec![
+ ContentItem::InputImage {
+ image: ImageReference::File {
+ file_id: "file_message".to_string(),
+ },
+ detail: Some(ImageDetail::High),
+ },
ContentItem::InputImage {
image: ImageReference::Inline {
image_url: small_image_url,
@@ -259,6 +266,7 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
ContentItemKind("user.image".to_string()),
ContentItemKind("user.image".to_string()),
ContentItemKind("user.image".to_string()),
+ ContentItemKind("user.image".to_string()),
]),
..Default::default()
},
@@ -270,6 +278,12 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
name: None,
namespace: None,
output: FunctionCallOutputPayload::from_content_items(vec![
+ FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::File {
+ file_id: "file_tool".to_string(),
+ },
+ detail: Some(ImageDetail::High),
+ },
FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline {
image_url: "data:image/png;base64,%%%".to_string(),
@@ -294,7 +308,7 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
);
let expected_user_notice = concat!(
"<image_resize_notice>\n",
- "Image 3 of 3 in the preceding user message was resized from 2048x2048 to 1600x1600 pixels.\n",
+ "Image 4 of 4 in the preceding user message was resized from 2048x2048 to 1600x1600 pixels.\n",
"</image_resize_notice>"
);
@@ -310,6 +324,7 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
internal_chat_message_metadata_passthrough,
&Some(InternalChatMessageMetadataPassthrough {
content_item_kinds: Some(vec![
+ ContentItemKind("user.image".to_string()),
ContentItemKind("user.image".to_string()),
ContentItemKind("images.preparation_error".to_string()),
ContentItemKind("user.image".to_string()),
@@ -318,6 +333,10 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
}),
);
let [
+ ContentItem::InputImage {
+ image: ImageReference::File { file_id },
+ ..
+ },
ContentItem::InputImage {
image:
ImageReference::Inline {
@@ -337,8 +356,9 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
},
] = content.as_slice()
else {
- panic!("expected unchanged image, failed image placeholder, and resized image");
+ panic!("expected file image, unchanged image, failed image placeholder, and resized image");
};
+ assert_eq!(file_id, "file_message");
assert_eq!(
decoded_image(small_message_image_url).1.dimensions(),
(64, 32)
@@ -373,6 +393,10 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
panic!("expected function call output");
};
let [
+ FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::File { file_id },
+ ..
+ },
FunctionCallOutputContentItem::InputText {
text: failed_tool_image,
},
@@ -385,8 +409,9 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
},
] = output.content_items().expect("tool output content items")
else {
- panic!("expected failed image placeholder and resized image in the tool output");
+ panic!("expected file image, failed image placeholder, and resized image in tool output");
};
+ assert_eq!(file_id, "file_tool");
assert_eq!(failed_tool_image, IMAGE_PROCESSING_ERROR_PLACEHOLDER);
assert_eq!(
decoded_image(resized_tool_image_url).1.dimensions(),
@@ -400,7 +425,7 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() {
content: vec![ContentItem::InputText {
text: concat!(
"<image_resize_notice>\n",
- "Image 2 of 2 in the preceding tool output was resized from 2048x2048 to 1600x1600 pixels.\n",
+ "Image 3 of 3 in the preceding tool output was resized from 2048x2048 to 1600x1600 pixels.\n",
"</image_resize_notice>"
)
.to_string(),
diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs
index e4396a92c6..00e7e935ca 100644
--- a/codex-rs/core/src/session/tests.rs
+++ b/codex-rs/core/src/session/tests.rs
@@ -11306,7 +11306,9 @@ async fn task_finish_emits_turn_item_lifecycle_for_leftover_pending_user_input()
text_elements: vec![text_element.clone()],
},
UserInput::Image {
- image_url: image_url.clone(),
+ image: ImageReference::Inline {
+ image_url: image_url.clone(),
+ },
detail: Some(ImageDetail::High),
},
];
diff --git a/codex-rs/core/src/session/tests/guardian_tests.rs b/codex-rs/core/src/session/tests/guardian_tests.rs
index eef4b2cf47..371a1ba0db 100644
--- a/codex-rs/core/src/session/tests/guardian_tests.rs
+++ b/codex-rs/core/src/session/tests/guardian_tests.rs
@@ -40,6 +40,7 @@ use codex_protocol::config_types::ApprovalsReviewer;
use codex_protocol::error::SandboxErr;
use codex_protocol::models::AdditionalPermissionProfile as PermissionProfile;
use codex_protocol::models::ContentItemKind;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::NetworkPermissions;
use codex_protocol::models::ResponseInputItem;
use codex_protocol::protocol::AskForApproval;
@@ -235,7 +236,9 @@ async fn request_permissions_routes_to_guardian_when_reviewer_is_enabled() {
.thread_extension_data
.get_or_init(crate::context::NodeReplReviewEvidence::default);
let image = UserInput::Image {
- image_url: image_url.to_string(),
+ image: ImageReference::Inline {
+ image_url: image_url.to_string(),
+ },
detail: None,
};
evidence.record("js", "cell", "image", vec![image]);
diff --git a/codex-rs/core/src/tools/handlers/mcp.rs b/codex-rs/core/src/tools/handlers/mcp.rs
index 1d00595a96..509cab24a8 100644
--- a/codex-rs/core/src/tools/handlers/mcp.rs
+++ b/codex-rs/core/src/tools/handlers/mcp.rs
@@ -367,7 +367,10 @@ impl CoreToolRuntime for McpHandler {
load_data_url_for_prompt_uncached(&image_url, PromptImageMode::Original)
.ok()?;
captured_image_bytes = next_image_bytes;
- Some(UserInput::Image { image_url, detail })
+ Some(UserInput::Image {
+ image: codex_protocol::models::ImageReference::Inline { image_url },
+ detail,
+ })
}
_ => None,
}
diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs
index 5016a06375..d8f6bab6ec 100644
--- a/codex-rs/core/tests/suite/compact.rs
+++ b/codex-rs/core/tests/suite/compact.rs
@@ -17,6 +17,7 @@ use codex_protocol::config_types::ReasoningSummary;
use codex_protocol::config_types::ServiceTier;
use codex_protocol::config_types::Settings;
use codex_protocol::items::TurnItem;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::PermissionProfile;
use codex_protocol::openai_models::ModelInfo;
use codex_protocol::openai_models::ModelServiceTier;
@@ -4833,7 +4834,9 @@ async fn snapshot_request_shape_pre_turn_compaction_including_incoming_user_mess
codex
.start_or_steer_turn(TurnInputRequest::user_input(vec![
UserInput::Image {
- image_url: image_url.clone(),
+ image: ImageReference::Inline {
+ image_url: image_url.clone(),
+ },
detail: None,
},
UserInput::Text {
diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs
index 53dee6ea97..82ed5f5092 100644
--- a/codex-rs/core/tests/suite/compact_remote.rs
+++ b/codex-rs/core/tests/suite/compact_remote.rs
@@ -17,6 +17,7 @@ use codex_model_provider_info::AMAZON_BEDROCK_PROVIDER_ID;
use codex_model_provider_info::ModelProviderInfo;
use codex_protocol::AgentPath;
use codex_protocol::models::ContentItem;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::ConversationStartParams;
use codex_protocol::protocol::EventMsg;
@@ -637,10 +638,12 @@ async fn remote_compact_v2_charges_retained_images_to_token_budget(
let mut bytes = std::io::Cursor::new(Vec::new());
image.write_to(&mut bytes, image::ImageFormat::Png)?;
Ok(UserInput::Image {
- image_url: format!(
- "data:image/png;base64,{}",
- BASE64_STANDARD.encode(bytes.get_ref())
- ),
+ image: ImageReference::Inline {
+ image_url: format!(
+ "data:image/png;base64,{}",
+ BASE64_STANDARD.encode(bytes.get_ref())
+ ),
+ },
detail: Some(codex_protocol::models::ImageDetail::Original),
})
})
@@ -714,7 +717,11 @@ async fn remote_compact_v2_charges_retained_images_to_token_budget(
};
let mut expected_images = prepared_images[dropped..].to_vec();
if cycle == 2 {
- let UserInput::Image { image_url, .. } = &image_inputs[7] else {
+ let UserInput::Image {
+ image: ImageReference::Inline { image_url },
+ ..
+ } = &image_inputs[7]
+ else {
unreachable!()
};
expected_images.push(image_url.clone());
diff --git a/codex-rs/core/tests/suite/context_annotations.rs b/codex-rs/core/tests/suite/context_annotations.rs
index aeb4a52882..34392c457e 100644
--- a/codex-rs/core/tests/suite/context_annotations.rs
+++ b/codex-rs/core/tests/suite/context_annotations.rs
@@ -4,6 +4,7 @@ use codex_core::config::CurrentTimeReminderConfig;
use codex_core::config::RolloutBudgetConfig;
use codex_core::config::TokenBudgetConfig;
use codex_features::Feature;
+use codex_protocol::models::ImageReference;
use codex_protocol::openai_models::InputModality;
use codex_protocol::protocol::AdditionalContextEntry;
use codex_protocol::protocol::AdditionalContextKind;
@@ -112,7 +113,9 @@ async fn first_request_item_types_roles_and_content_annotations() -> Result<()>
text_elements: Vec::new(),
},
UserInput::Image {
- image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==".to_string(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==".to_string(),
+ },
detail: None,
},
UserInput::Audio {
diff --git a/codex-rs/core/tests/suite/guardian_authorization.rs b/codex-rs/core/tests/suite/guardian_authorization.rs
index 9a8ccfbf55..a252baee34 100644
--- a/codex-rs/core/tests/suite/guardian_authorization.rs
+++ b/codex-rs/core/tests/suite/guardian_authorization.rs
@@ -11,6 +11,7 @@ use codex_core::context::InternalModelContextFragment;
use codex_features::Feature;
use codex_history::RolloutItem;
use codex_protocol::config_types::ApprovalsReviewer;
+use codex_protocol::models::ImageReference;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::GuardianAssessmentStatus;
@@ -328,7 +329,9 @@ async fn guardian_authorization_revision_survives_compaction_not_user_input() ->
.await;
test.codex
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Image {
- image_url: "data:image/png;base64,not-an-image".to_owned(),
+ image: ImageReference::Inline {
+ image_url: "data:image/png;base64,not-an-image".to_owned(),
+ },
detail: None,
}]))
.await?;
diff --git a/codex-rs/core/tests/suite/guardian_history.rs b/codex-rs/core/tests/suite/guardian_history.rs
index 5ccc0fa328..94badeabc3 100644
--- a/codex-rs/core/tests/suite/guardian_history.rs
+++ b/codex-rs/core/tests/suite/guardian_history.rs
@@ -13,6 +13,7 @@ use codex_history::ResumedHistory;
use codex_history::RolloutItem;
use codex_protocol::config_types::ApprovalsReviewer;
use codex_protocol::mcp::ClientMcpExtensions;
+use codex_protocol::models::ImageReference;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::Op;
@@ -441,7 +442,9 @@ async fn guardian_history_survives_compaction_and_eviction_but_not_legacy_rollba
text_elements: Vec::new(),
},
UserInput::Image {
- image_url: image_url.clone(),
+ image: ImageReference::Inline {
+ image_url: image_url.clone(),
+ },
detail: None,
},
]))
diff --git a/codex-rs/core/tests/suite/image_rollout.rs b/codex-rs/core/tests/suite/image_rollout.rs
index e12d6a7f57..e9145c1d5d 100644
--- a/codex-rs/core/tests/suite/image_rollout.rs
+++ b/codex-rs/core/tests/suite/image_rollout.rs
@@ -232,7 +232,9 @@ async fn drag_drop_image_persists_rollout_request_shape() -> anyhow::Result<()>
.start_or_steer_turn(
TurnInputRequest::user_input(vec![
UserInput::Image {
- image_url: image_url.clone(),
+ image: ImageReference::Inline {
+ image_url: image_url.clone(),
+ },
detail: None,
},
UserInput::Text {
@@ -289,6 +291,83 @@ async fn drag_drop_image_persists_rollout_request_shape() -> anyhow::Result<()>
Ok(())
}
+/// Core must forward an opaque file ID and persist that same reference in canonical history.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn file_image_passes_through_request_and_rollout() -> anyhow::Result<()> {
+ skip_if_no_network!(Ok(()));
+
+ let server = start_mock_server().await;
+ let test = test_codex().build_with_auto_env(&server).await?;
+ let response_mock = responses::mount_sse_once(
+ &server,
+ sse(vec![
+ ev_response_created("resp-file"),
+ ev_assistant_message("msg-file", "done"),
+ ev_completed("resp-file"),
+ ]),
+ )
+ .await;
+
+ test.codex
+ .start_or_steer_turn(TurnInputRequest::user_input(vec![
+ UserInput::Image {
+ image: ImageReference::File {
+ file_id: "file_123".to_string(),
+ },
+ detail: None,
+ },
+ UserInput::Text {
+ text: "file-backed image".to_string(),
+ text_elements: Vec::new(),
+ },
+ ]))
+ .await?;
+ wait_for_event(&test.codex, |event| {
+ matches!(event, EventMsg::TurnComplete(_))
+ })
+ .await;
+
+ let request = response_mock.single_request();
+ assert!(request.input().iter().any(|item| {
+ item.get("content")
+ .and_then(Value::as_array)
+ .is_some_and(|content| {
+ content.iter().any(|item| {
+ item.get("type").and_then(Value::as_str) == Some("input_image")
+ && item.get("file_id").and_then(Value::as_str) == Some("file_123")
+ })
+ })
+ }));
+
+ test.codex.shutdown_and_wait().await?;
+ let rollout_path = test.codex.rollout_path().expect("rollout path");
+ let rollout_text = read_rollout_text(&rollout_path).await?;
+ let actual = find_user_message_with_image(&rollout_text)
+ .expect("expected user message with file image in rollout");
+ assert_eq!(
+ strip_response_item_id(strip_metadata(actual)),
+ ResponseItem::Message {
+ id: None,
+ role: "user".to_string(),
+ content: vec![
+ ContentItem::InputImage {
+ image: ImageReference::File {
+ file_id: "file_123".to_string(),
+ },
+ detail: Some(DEFAULT_IMAGE_DETAIL),
+ },
+ ContentItem::InputText {
+ text: "file-backed image".to_string(),
+ },
+ ],
+ phase: None,
+ internal_chat_message_metadata_passthrough: None,
+ }
+ );
+
+ Ok(())
+}
+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
@@ -379,7 +458,9 @@ async fn resumed_history_only_emits_resize_notices_for_new_images() -> anyhow::R
resumed
.codex
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Image {
- image_url: original_image_url.clone(),
+ image: ImageReference::Inline {
+ image_url: original_image_url.clone(),
+ },
detail: Some(ImageDetail::High),
}]))
.await?;
diff --git a/codex-rs/core/tests/suite/items.rs b/codex-rs/core/tests/suite/items.rs
index a3d3d44857..f50baf2ad0 100644
--- a/codex-rs/core/tests/suite/items.rs
+++ b/codex-rs/core/tests/suite/items.rs
@@ -9,6 +9,8 @@ use codex_protocol::config_types::ModeKind;
use codex_protocol::config_types::Settings;
use codex_protocol::items::AgentMessageContent;
use codex_protocol::items::TurnItem;
+use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::WebSearchAction;
use codex_protocol::protocol::AskForApproval;
@@ -81,13 +83,21 @@ async fn user_message_item_is_emitted() -> anyhow::Result<()> {
ByteRange { start: 0, end: 6 },
Some("<file>".into()),
)];
- let expected_input = UserInput::Text {
- text: "please inspect sample.txt".into(),
- text_elements: text_elements.clone(),
- };
+ let expected_input = vec![
+ UserInput::Text {
+ text: "please inspect sample.txt".into(),
+ text_elements: text_elements.clone(),
+ },
+ UserInput::Image {
+ image: ImageReference::File {
+ file_id: "file_123".into(),
+ },
+ detail: Some(ImageDetail::High),
+ },
+ ];
codex
- .start_or_steer_turn(TurnInputRequest::user_input(vec![expected_input.clone()]))
+ .start_or_steer_turn(TurnInputRequest::user_input(expected_input.clone()))
.await?;
let started_item = wait_for_event_match(&codex, |ev| match ev {
@@ -108,8 +118,8 @@ async fn user_message_item_is_emitted() -> anyhow::Result<()> {
.await;
assert_eq!(started_item.id, completed_item.id);
- assert_eq!(started_item.content, vec![expected_input.clone()]);
- assert_eq!(completed_item.content, vec![expected_input]);
+ assert_eq!(started_item.content, expected_input);
+ assert_eq!(completed_item.content, started_item.content);
let legacy_message = wait_for_event_match(&codex, |ev| match ev {
EventMsg::UserMessage(event) => Some(event.clone()),
@@ -118,6 +128,12 @@ async fn user_message_item_is_emitted() -> anyhow::Result<()> {
.await;
assert_eq!(legacy_message.message, "please inspect sample.txt");
assert_eq!(legacy_message.text_elements, text_elements);
+ assert_eq!(legacy_message.file_ids, Some(vec!["file_123".into()]));
+ assert_eq!(
+ legacy_message.file_id_details,
+ vec![Some(ImageDetail::High)]
+ );
+
Ok(())
}
diff --git a/codex-rs/core/tests/suite/model_switching.rs b/codex-rs/core/tests/suite/model_switching.rs
index 70e804ee1e..a7a4493e94 100644
--- a/codex-rs/core/tests/suite/model_switching.rs
+++ b/codex-rs/core/tests/suite/model_switching.rs
@@ -18,6 +18,7 @@ use codex_protocol::config_types::ServiceTier;
use codex_protocol::config_types::Settings;
use codex_protocol::mcp::ClientMcpExtensions;
use codex_protocol::models::BaseInstructionsProvenance;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::PermissionProfile;
use codex_protocol::openai_models::ConfigShellToolType;
use codex_protocol::openai_models::InputModality;
@@ -906,7 +907,9 @@ async fn model_change_projects_media_without_changing_live_or_replayed_history(
&test,
vec![
UserInput::Image {
- image_url: image_url.clone(),
+ image: ImageReference::Inline {
+ image_url: image_url.clone(),
+ },
detail: None,
},
UserInput::Audio {
diff --git a/codex-rs/core/tests/suite/responses_lite.rs b/codex-rs/core/tests/suite/responses_lite.rs
index 8ffe25e982..c35be9664b 100644
--- a/codex-rs/core/tests/suite/responses_lite.rs
+++ b/codex-rs/core/tests/suite/responses_lite.rs
@@ -14,6 +14,7 @@ use codex_login::auth::BedrockApiKeyAuth;
use codex_mcp::CODEX_APPS_MCP_SERVER_NAME;
use codex_protocol::config_types::WebSearchMode;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::openai_models::InputModality;
use codex_protocol::openai_models::ToolMode;
use codex_protocol::protocol::EventMsg;
@@ -278,11 +279,15 @@ async fn responses_lite_prepares_images() -> Result<()> {
test.codex
.start_or_steer_turn(TurnInputRequest::user_input(vec![
UserInput::Image {
- image_url: image_url.to_string(),
+ image: ImageReference::Inline {
+ image_url: image_url.to_string(),
+ },
detail: Some(ImageDetail::Original),
},
UserInput::Image {
- image_url: remote_image_url.to_string(),
+ image: ImageReference::Inline {
+ image_url: remote_image_url.to_string(),
+ },
detail: Some(ImageDetail::High),
},
]))
diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs
index 23405bf9fc..eaa89dd658 100644
--- a/codex-rs/core/tests/suite/rmcp_client.rs
+++ b/codex-rs/core/tests/suite/rmcp_client.rs
@@ -1488,7 +1488,9 @@ async fn interrupt_during_mcp_startup_preserves_user_input_in_history(
unreachable!("read_only_user_turn creates user input");
};
content.push(UserInput::Image {
- image_url: OPENAI_PNG.to_string(),
+ image: ImageReference::Inline {
+ image_url: OPENAI_PNG.to_string(),
+ },
detail: Some(ImageDetail::High),
});
fixture.codex.start_or_steer_turn(input).await?;
diff --git a/codex-rs/core/tests/suite/scenarios.rs b/codex-rs/core/tests/suite/scenarios.rs
index d339e5b16d..a54147e9e0 100644
--- a/codex-rs/core/tests/suite/scenarios.rs
+++ b/codex-rs/core/tests/suite/scenarios.rs
@@ -23,6 +23,7 @@ use codex_login::CodexAuth;
use codex_models_manager::bundled_models_response;
use codex_protocol::items::AgentMessageDelivery;
use codex_protocol::items::TurnItem;
+use codex_protocol::models::ImageReference;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::Op;
use codex_protocol::user_input::UserInput;
@@ -382,7 +383,9 @@ async fn astra_kickoff_with_skills_plugins_and_remote_compaction() -> Result<()>
.start_or_steer_turn(TurnInputRequest::user_input(vec![
text("Check the final kickoff brief and attached sketch with $final-check and $calendar:agenda."),
UserInput::Image {
- image_url: format!("data:image/png;base64,{ONE_PIXEL_PNG_BASE64}"),
+ image: ImageReference::Inline {
+ image_url: format!("data:image/png;base64,{ONE_PIXEL_PNG_BASE64}"),
+ },
detail: None,
},
selected_skill("final-check", &skills.final_check),
diff --git a/codex-rs/ext/image-generation/src/tests.rs b/codex-rs/ext/image-generation/src/tests.rs
index 32b0493775..791244445e 100644
--- a/codex-rs/ext/image-generation/src/tests.rs
+++ b/codex-rs/ext/image-generation/src/tests.rs
@@ -166,6 +166,87 @@ async fn recent_image_fallback_selects_newest_images_in_chronological_order() {
);
}
+/// A file-backed image in the requested window must not cause an older inline image to be edited.
+#[tokio::test]
+async fn recent_image_fallback_rejects_file_backed_image_in_requested_window() {
+ let error = request_for_call_args(
+ &ImagegenArgs {
+ prompt: "change the lighting".to_string(),
+ referenced_image_paths: None,
+ num_last_images_to_include: Some(1),
+ },
+ &[ResponseItem::Message {
+ id: None,
+ role: "user".to_string(),
+ content: vec![
+ input_image("older-inline-image"),
+ ContentItem::InputImage {
+ image: ImageReference::File {
+ file_id: "newer-file-backed-image".to_string(),
+ },
+ detail: None,
+ },
+ ],
+ phase: None,
+ internal_chat_message_metadata_passthrough: None,
+ }],
+ &[],
+ )
+ .await
+ .expect_err("a selected file-backed image should fail");
+
+ assert_eq!(
+ error.to_string(),
+ "requested the last 1 conversation images, but that window includes a file-backed image \
+ that cannot be used for editing"
+ );
+}
+
+/// Tool-output file references must count toward the window instead of exposing an older image.
+#[tokio::test]
+async fn recent_image_fallback_rejects_file_backed_tool_output_in_requested_window() {
+ let error = request_for_call_args(
+ &ImagegenArgs {
+ prompt: "change the lighting".to_string(),
+ referenced_image_paths: None,
+ num_last_images_to_include: Some(1),
+ },
+ &[
+ ResponseItem::FunctionCallOutput {
+ id: None,
+ call_id: Some("older-inline-output".to_string()),
+ name: None,
+ namespace: None,
+ output: image_output("older-inline-image"),
+ internal_chat_message_metadata_passthrough: None,
+ },
+ ResponseItem::CustomToolCallOutput {
+ id: None,
+ call_id: "newer-file-backed-output".to_string(),
+ name: Some("view_image".to_string()),
+ output: FunctionCallOutputPayload::from_content_items(vec![
+ FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::File {
+ file_id: "newer-file-backed-image".to_string(),
+ },
+ detail: None,
+ },
+ ]),
+ internal_chat_message_metadata_passthrough: None,
+ },
+ ],
+ &[],
+ )
+ .await
+ .expect_err("a selected file-backed tool output should fail");
+
+ assert_eq!(
+ error.to_string(),
+ "requested the last 1 conversation images, but that window includes a file-backed image \
+ that cannot be used for editing"
+ );
+}
+
#[tokio::test]
async fn conflicting_image_selectors_return_tool_error() {
let error = request_for_call_args(
diff --git a/codex-rs/ext/image-generation/src/tool.rs b/codex-rs/ext/image-generation/src/tool.rs
index 8fc408b442..66864fea6b 100644
--- a/codex-rs/ext/image-generation/src/tool.rs
+++ b/codex-rs/ext/image-generation/src/tool.rs
@@ -459,14 +459,7 @@ async fn request_for_call_args(
}
// Pathless images have no stable reference, so this bounded window may include newer
// unrelated images. This remains best-effort until the harness provides stable refs.
- let images = recent_images(history, count);
- if images.len() != count {
- return Err(FunctionCallError::RespondToModel(format!(
- "requested the last {count} conversation images, but only {} were available",
- images.len()
- )));
- }
- images
+ recent_images(history, count)?
}
(false, Some(_)) => {
return Err(FunctionCallError::RespondToModel(
@@ -488,17 +481,19 @@ async fn request_for_call_args(
}))
}
-fn recent_images(history: &[ResponseItem], count: usize) -> Vec<ImageUrl> {
+fn recent_images(
+ history: &[ResponseItem],
+ count: usize,
+) -> Result<Vec<ImageUrl>, FunctionCallError> {
let mut images = Vec::with_capacity(count);
'history: for item in history.iter().rev() {
let mut image_urls = Vec::new();
match item {
ResponseItem::Message { content, .. } => {
image_urls.extend(content.iter().rev().filter_map(|item| match item {
- ContentItem::InputImage {
- image: ImageReference::Inline { image_url },
- ..
- } => Some(image_url.clone()),
+ ContentItem::InputImage { image, .. } => {
+ Some(inline_image_url(image).map(str::to_owned))
+ }
ContentItem::InputText { .. }
| ContentItem::InputAudio { .. }
| ContentItem::OutputText { .. } => None,
@@ -506,10 +501,12 @@ fn recent_images(history: &[ResponseItem], count: usize) -> Vec<ImageUrl> {
}
ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. } => {
- image_urls.extend(output_image_urls(output));
+ image_urls.extend(
+ output_images(output).map(|image| inline_image_url(image).map(str::to_owned)),
+ );
}
ResponseItem::ImageGenerationCall { result, .. } if !result.is_empty() => {
- image_urls.push(format!("data:image/png;base64,{result}"));
+ image_urls.push(Some(format!("data:image/png;base64,{result}")));
}
ResponseItem::AdditionalTools { .. }
| ResponseItem::Reasoning { .. }
@@ -528,34 +525,55 @@ fn recent_images(history: &[ResponseItem], count: usize) -> Vec<ImageUrl> {
| ResponseItem::Other => {}
}
for image_url in image_urls {
- images.push(ImageUrl { image_url });
+ images.push(image_url.map(|image_url| ImageUrl { image_url }));
if images.len() == count {
break 'history;
}
}
}
- images.reverse();
- images
+ if images.len() != count {
+ return Err(FunctionCallError::RespondToModel(format!(
+ "requested the last {count} conversation images, but only {} were available",
+ images.len()
+ )));
+ }
+ let mut inline_images = Vec::with_capacity(count);
+ for image in images {
+ let Some(image) = image else {
+ return Err(FunctionCallError::RespondToModel(format!(
+ "requested the last {count} conversation images, but that window includes a \
+ file-backed image that cannot be used for editing"
+ )));
+ };
+ inline_images.push(image);
+ }
+ inline_images.reverse();
+ Ok(inline_images)
}
-/// Extracts image URLs from a tool output in newest-first order.
-fn output_image_urls(output: &FunctionCallOutputPayload) -> impl Iterator<Item = String> + '_ {
+/// Extracts image references from a tool output in newest-first order.
+fn output_images(output: &FunctionCallOutputPayload) -> impl Iterator<Item = &ImageReference> + '_ {
output
.content_items()
.into_iter()
.flatten()
.rev()
.filter_map(|item| match item {
- FunctionCallOutputContentItem::InputImage {
- image: ImageReference::Inline { image_url },
- ..
- } => Some(image_url.clone()),
+ FunctionCallOutputContentItem::InputImage { image, .. } => Some(image),
FunctionCallOutputContentItem::InputText { .. }
| FunctionCallOutputContentItem::InputAudio { .. }
| FunctionCallOutputContentItem::EncryptedContent { .. } => None,
})
}
+fn inline_image_url(image: &ImageReference) -> Option<&str> {
+ match image {
+ ImageReference::Inline { image_url } => Some(image_url),
+ // TODO(kc) Image generation and the Images API only accept inline image URLs.
+ ImageReference::File { .. } => None,
+ }
+}
+
async fn image_url(
path: &AbsolutePathBuf,
environment: &ToolEnvironment<'_>,
diff --git a/codex-rs/ext/queue/tests/queue_service.rs b/codex-rs/ext/queue/tests/queue_service.rs
index a597583d39..cadd2fc7fc 100644
--- a/codex-rs/ext/queue/tests/queue_service.rs
+++ b/codex-rs/ext/queue/tests/queue_service.rs
@@ -29,6 +29,7 @@ use codex_extension_api::TurnStartAdmission;
use codex_protocol::ThreadId;
use codex_protocol::items::TurnItem;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::Event;
use codex_protocol::protocol::EventMsg;
@@ -825,7 +826,9 @@ async fn local_attachments_are_snapshotted_before_enqueue_and_update() -> anyhow
let expected_queued_input = user_input_with_media(
"queued attachments",
UserInput::Image {
- image_url: TINY_PNG_DATA_URL.to_string(),
+ image: ImageReference::Inline {
+ image_url: TINY_PNG_DATA_URL.to_string(),
+ },
detail: Some(ImageDetail::Original),
},
UserInput::Audio {
@@ -865,7 +868,9 @@ async fn local_attachments_are_snapshotted_before_enqueue_and_update() -> anyhow
let expected_edited_input = user_input_with_media(
"edited attachments",
UserInput::Image {
- image_url: TINY_PNG_DATA_URL.to_string(),
+ image: ImageReference::Inline {
+ image_url: TINY_PNG_DATA_URL.to_string(),
+ },
detail: Some(ImageDetail::High),
},
UserInput::Audio {
diff --git a/codex-rs/guardian-context/src/budget.rs b/codex-rs/guardian-context/src/budget.rs
index b29ff2c66f..1ebec846e4 100644
--- a/codex-rs/guardian-context/src/budget.rs
+++ b/codex-rs/guardian-context/src/budget.rs
@@ -105,6 +105,10 @@ impl SectionCost {
self.image_bytes = self.image_bytes.saturating_add(image_url.len());
self.image_count = self.image_count.saturating_add(1);
}
+ ContentItem::InputImage {
+ image: ImageReference::File { .. },
+ ..
+ } => {}
ContentItem::InputAudio { audio_url } => {
// Guardian currently has no audio contributor. Count a future opaque
// payload conservatively until its consumer supplies modality costs.
diff --git a/codex-rs/guardian-context/src/composition.rs b/codex-rs/guardian-context/src/composition.rs
index 555fbb2163..66892875fd 100644
--- a/codex-rs/guardian-context/src/composition.rs
+++ b/codex-rs/guardian-context/src/composition.rs
@@ -210,20 +210,27 @@ impl CollectedContext {
let items = evidence
.items
.into_iter()
- .map(|item| match item {
+ .filter_map(|item| match item {
UserInput::Text { text, .. } => {
- Ok(Budgeted::required(ContentItem::InputText { text }))
+ Some(Ok(Budgeted::required(ContentItem::InputText { text })))
}
- UserInput::Image { image_url, detail } => Ok(Budgeted::optional(
+ UserInput::Image {
+ image: ImageReference::Inline { image_url },
+ detail,
+ } => Some(Ok(Budgeted::optional(
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
},
BudgetPriority::Image,
- )),
- _ => Err(SectionError::UnsupportedDelivery {
+ ))),
+ UserInput::Image {
+ image: ImageReference::File { .. },
+ ..
+ } => None,
+ _ => Some(Err(SectionError::UnsupportedDelivery {
section: "node_repl_evidence",
- }),
+ })),
})
.collect::<Result<Vec<_>, _>>()?;
(
@@ -288,7 +295,14 @@ impl ComposedContext {
ContentItem::InputImage {
image: ImageReference::Inline { image_url },
detail,
- } => UserInput::Image { image_url, detail },
+ } => UserInput::Image {
+ image: ImageReference::Inline { image_url },
+ detail,
+ },
+ ContentItem::InputImage {
+ image: ImageReference::File { .. },
+ ..
+ } => continue,
ContentItem::InputAudio { .. } | ContentItem::OutputText { .. } => {
return Err(SectionError::UnsupportedDelivery {
section: section.id,
@@ -316,6 +330,10 @@ impl ComposedContext {
}
}));
}
+ ContentItem::InputImage {
+ image: ImageReference::File { .. },
+ ..
+ } => {}
content => user_content.push(content),
}
}
diff --git a/codex-rs/guardian-context/src/enforcement.rs b/codex-rs/guardian-context/src/enforcement.rs
index 54cf00d076..4e59d73463 100644
--- a/codex-rs/guardian-context/src/enforcement.rs
+++ b/codex-rs/guardian-context/src/enforcement.rs
@@ -95,6 +95,10 @@ impl ComposedContext {
image: ImageReference::Inline { image_url },
detail,
} => admit(image_url, detail),
+ ContentItem::InputImage {
+ image: ImageReference::File { .. },
+ ..
+ } => false,
_ => true,
});
}
@@ -281,6 +285,10 @@ fn retain_content(
image: ImageReference::Inline { image_url },
..
} => image_url.len(),
+ ContentItem::InputImage {
+ image: ImageReference::File { .. },
+ ..
+ } => 0,
ContentItem::InputAudio { audio_url } => audio_url.len(),
};
truncations.push(TruncationObservation {
diff --git a/codex-rs/guardian-context/src/images.rs b/codex-rs/guardian-context/src/images.rs
index ccae031095..27314df4db 100644
--- a/codex-rs/guardian-context/src/images.rs
+++ b/codex-rs/guardian-context/src/images.rs
@@ -85,18 +85,26 @@ impl TranscriptImages {
});
};
+ // TODO(kc) Include file-backed images once Guardian can admit them without resolving or
+ // applying byte-based checks in Core.
for item in items {
match item {
ResponseItem::Message { role, content, .. }
if matches!(role.as_str(), "user" | "assistant") =>
{
for item in content {
- if let ContentItem::InputImage {
- image: ImageReference::Inline { image_url },
- detail,
- } = item
- {
- include_image(image_url, *detail);
+ match item {
+ ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail,
+ } => include_image(image_url, *detail),
+ ContentItem::InputImage {
+ image: ImageReference::File { .. },
+ ..
+ }
+ | ContentItem::InputText { .. }
+ | ContentItem::InputAudio { .. }
+ | ContentItem::OutputText { .. } => {}
}
}
}
@@ -106,12 +114,18 @@ impl TranscriptImages {
{
if let Some(content) = output.content_items() {
for item in content {
- if let FunctionCallOutputContentItem::InputImage {
- image: ImageReference::Inline { image_url },
- detail,
- } = item
- {
- include_image(image_url, *detail);
+ match item {
+ FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail,
+ } => include_image(image_url, *detail),
+ FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::File { .. },
+ ..
+ }
+ | FunctionCallOutputContentItem::InputText { .. }
+ | FunctionCallOutputContentItem::InputAudio { .. }
+ | FunctionCallOutputContentItem::EncryptedContent { .. } => {}
}
}
}
@@ -121,12 +135,18 @@ impl TranscriptImages {
}
if input.include_tool_outputs {
for image in input.node_repl_images {
- if let ContentItem::InputImage {
- image: ImageReference::Inline { image_url },
- detail,
- } = image
- {
- include_image(image_url, *detail);
+ match image {
+ ContentItem::InputImage {
+ image: ImageReference::Inline { image_url },
+ detail,
+ } => include_image(image_url, *detail),
+ ContentItem::InputImage {
+ image: ImageReference::File { .. },
+ ..
+ }
+ | ContentItem::InputText { .. }
+ | ContentItem::InputAudio { .. }
+ | ContentItem::OutputText { .. } => {}
}
}
}
diff --git a/codex-rs/guardian-context/src/images_tests.rs b/codex-rs/guardian-context/src/images_tests.rs
index 090b5c0531..5c3fa54d04 100644
--- a/codex-rs/guardian-context/src/images_tests.rs
+++ b/codex-rs/guardian-context/src/images_tests.rs
@@ -98,3 +98,69 @@ fn image_selection_preserves_source_policy_order_and_both_limits() {
}
);
}
+
+/// File-backed images stay out of Guardian without consuming its inline-image capacity.
+#[test]
+fn file_images_are_omitted_without_displacing_inline_images() {
+ let file_image = ContentItem::InputImage {
+ image: ImageReference::File {
+ file_id: "file_123".to_string(),
+ },
+ detail: Some(ImageDetail::High),
+ };
+ let history = [
+ ResponseItem::Message {
+ id: None,
+ role: "user".into(),
+ content: vec![image("first"), file_image.clone(), image("second")],
+ phase: None,
+ internal_chat_message_metadata_passthrough: None,
+ },
+ ResponseItem::FunctionCallOutput {
+ id: None,
+ call_id: Some("screenshot".into()),
+ name: None,
+ namespace: None,
+ output: FunctionCallOutputPayload::from_content_items(vec![
+ FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::File {
+ file_id: "file_tool".to_string(),
+ },
+ detail: Some(ImageDetail::High),
+ },
+ FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::Inline {
+ image_url: "third".into(),
+ },
+ detail: Some(ImageDetail::High),
+ },
+ ]),
+ internal_chat_message_metadata_passthrough: None,
+ },
+ ];
+ let repl = [
+ file_image,
+ image("fourth"),
+ ContentItem::InputImage {
+ image: ImageReference::File {
+ file_id: "file_repl".to_string(),
+ },
+ detail: Some(ImageDetail::High),
+ },
+ ];
+
+ assert_eq!(
+ TranscriptImages::collect(
+ &history,
+ TranscriptImageInput {
+ enabled: true,
+ include_tool_outputs: true,
+ node_repl_images: &repl,
+ }
+ ),
+ TranscriptImages {
+ images: ["first", "second", "third", "fourth"].map(image).to_vec(),
+ omitted_bytes: 0,
+ }
+ );
+}
diff --git a/codex-rs/guardian-context/src/node_repl.rs b/codex-rs/guardian-context/src/node_repl.rs
index 01b6797d7e..4e7c370379 100644
--- a/codex-rs/guardian-context/src/node_repl.rs
+++ b/codex-rs/guardian-context/src/node_repl.rs
@@ -9,6 +9,7 @@ use crate::SectionInput;
use crate::SectionScope;
use codex_context_fragments::ContextualUserFragment;
use codex_protocol::models::ContentItemKind;
+use codex_protocol::models::ImageReference;
use codex_protocol::user_input::UserInput;
use codex_protocol::user_input::UserInput::Image;
use codex_protocol::user_input::UserInput::Text;
@@ -84,10 +85,15 @@ impl NodeReplContext<'_> {
pub fn render_inputs(&self) -> Vec<UserInput> {
if self.mode != NodeReplReviewEvidenceMode::Multimodal
|| !self.responses.iter().any(|response| {
- response
- .items
- .iter()
- .any(|item| matches!(item, Image { .. }))
+ response.items.iter().any(|item| {
+ matches!(
+ item,
+ Image {
+ image: ImageReference::Inline { .. },
+ ..
+ }
+ )
+ })
})
{
return vec![text_input(self.render())];
@@ -138,9 +144,10 @@ impl NodeReplContext<'_> {
for item in response.items {
match item {
Text { text, .. } => inputs.push(text_input(format!("{text}\n"))),
- Image { image_url, .. } if seen_images.insert(image_url) => {
- inputs.push(item.clone())
- }
+ Image {
+ image: ImageReference::Inline { image_url },
+ ..
+ } if seen_images.insert(image_url) => inputs.push(item.clone()),
_ => {}
}
}
diff --git a/codex-rs/otel/tests/suite/otel_export_routing_policy.rs b/codex-rs/otel/tests/suite/otel_export_routing_policy.rs
index 7605fab73e..38c5eb49a7 100644
--- a/codex-rs/otel/tests/suite/otel_export_routing_policy.rs
+++ b/codex-rs/otel/tests/suite/otel_export_routing_policy.rs
@@ -24,6 +24,7 @@ use codex_protocol::ThreadId;
use codex_protocol::ToolName;
use codex_protocol::config_types::ReasoningSummary;
use codex_protocol::config_types::ToolResultLogConfig;
+use codex_protocol::models::ImageReference;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::SandboxPolicy;
use codex_protocol::protocol::SessionSource;
@@ -143,7 +144,9 @@ fn otel_export_routing_policy_routes_user_prompt_log_and_trace_events() {
text_elements: Vec::new(),
},
UserInput::Image {
- image_url: "https://example.com/image.png".to_string(),
+ image: ImageReference::Inline {
+ image_url: "https://example.com/image.png".to_string(),
+ },
detail: None,
},
UserInput::LocalImage {
diff --git a/codex-rs/protocol/src/items.rs b/codex-rs/protocol/src/items.rs
index b6c18412f4..2b35643072 100644
--- a/codex-rs/protocol/src/items.rs
+++ b/codex-rs/protocol/src/items.rs
@@ -7,6 +7,7 @@ use crate::memory_citation::MemoryCitation;
use crate::models::ContentItem;
use crate::models::FunctionCallOutputBody;
use crate::models::ImageDetail;
+use crate::models::ImageReference;
use crate::models::MessagePhase;
use crate::models::ResponseItem;
use crate::models::WebSearchAction;
@@ -554,7 +555,10 @@ impl UserMessageItem {
self.content
.iter()
.filter_map(|c| match c {
- UserInput::Image { image_url, .. } => Some(image_url.clone()),
+ UserInput::Image {
+ image: ImageReference::Inline { image_url },
+ ..
+ } => Some(image_url.clone()),
_ => None,
})
.collect()
@@ -565,7 +569,10 @@ impl UserMessageItem {
self.content
.iter()
.filter_map(|c| match c {
- UserInput::Image { detail, .. } => Some(*detail),
+ UserInput::Image {
+ image: ImageReference::Inline { .. },
+ detail,
+ } => Some(*detail),
_ => None,
})
.collect(),
@@ -615,7 +622,7 @@ impl UserMessageItem {
}
}
-fn trim_trailing_default_image_details(
+pub(crate) fn trim_trailing_default_image_details(
mut details: Vec<Option<ImageDetail>>,
) -> Vec<Option<ImageDetail>> {
while matches!(details.last(), Some(None)) {
diff --git a/codex-rs/protocol/src/legacy_events.rs b/codex-rs/protocol/src/legacy_events.rs
index bfea90b558..b8f73e0641 100644
--- a/codex-rs/protocol/src/legacy_events.rs
+++ b/codex-rs/protocol/src/legacy_events.rs
@@ -20,6 +20,8 @@ use crate::items::SubAgentActivityItem;
use crate::items::TurnItem;
use crate::items::UserMessageItem;
use crate::items::WebSearchItem;
+use crate::items::trim_trailing_default_image_details;
+use crate::models::ImageReference;
use crate::protocol::AgentMessageContentDeltaEvent;
use crate::protocol::AgentMessageEvent;
use crate::protocol::AgentReasoningEvent;
@@ -58,9 +60,11 @@ use crate::protocol::ReasoningContentDeltaEvent;
use crate::protocol::ReasoningRawContentDeltaEvent;
use crate::protocol::SubAgentActivityEvent;
use crate::protocol::UserMessageEvent;
+use crate::protocol::UserMessageImageKind;
use crate::protocol::ViewImageToolCallEvent;
use crate::protocol::WebSearchBeginEvent;
use crate::protocol::WebSearchEndEvent;
+use crate::user_input::UserInput;
/// Converts canonical item lifecycle events back into the legacy raw event stream used by
/// compatibility consumers that have not migrated to `TurnItem`.
@@ -78,11 +82,32 @@ impl UserMessageItem {
pub fn as_legacy_user_message_event(&self) -> UserMessageEvent {
// Legacy user-message events flatten only text inputs into `message` and
// rebase text element ranges onto that concatenated text.
+ let mut file_ids = Vec::new();
+ let mut file_id_details = Vec::new();
+ let mut image_order = Vec::new();
+ for input in &self.content {
+ if let UserInput::Image { image, detail } = input {
+ match image {
+ ImageReference::Inline { .. } => {
+ image_order.push(UserMessageImageKind::Inline);
+ }
+ ImageReference::File { file_id } => {
+ image_order.push(UserMessageImageKind::File);
+ file_ids.push(file_id.clone());
+ file_id_details.push(*detail);
+ }
+ }
+ }
+ }
+ let file_id_details = trim_trailing_default_image_details(file_id_details);
UserMessageEvent {
client_id: self.client_id.clone(),
message: self.message(),
images: Some(self.image_urls()),
image_details: self.image_details(),
+ file_ids: (!file_ids.is_empty()).then_some(file_ids),
+ file_id_details,
+ image_order,
local_images: self.local_image_paths(),
local_image_details: self.local_image_details(),
audio: Some(self.audio_urls()),
diff --git a/codex-rs/protocol/src/local_media.rs b/codex-rs/protocol/src/local_media.rs
index 7e4cbd9dfd..e1b8af25cf 100644
--- a/codex-rs/protocol/src/local_media.rs
+++ b/codex-rs/protocol/src/local_media.rs
@@ -8,6 +8,7 @@ use codex_utils_image::data_url_from_bytes;
use codex_utils_image::load_for_prompt_bytes;
use crate::models::ImageDetail;
+use crate::models::ImageReference;
use crate::user_input::UserInput;
/// Maximum accepted decoded byte length for prompt audio inputs.
@@ -29,7 +30,9 @@ pub fn snapshot_local_user_input(input: &mut UserInput) -> io::Result<()> {
let file_bytes = read_bounded_local_media(path, MAX_PROMPT_IMAGE_INPUT_BYTES, "image")?;
let image = load_for_prompt_bytes(path, file_bytes, mode).map_err(io::Error::other)?;
*input = UserInput::Image {
- image_url: image.into_data_url(),
+ image: ImageReference::Inline {
+ image_url: image.into_data_url(),
+ },
detail: image_detail,
};
}
diff --git a/codex-rs/protocol/src/local_media_tests.rs b/codex-rs/protocol/src/local_media_tests.rs
index 3d03c32620..47b28c8c25 100644
--- a/codex-rs/protocol/src/local_media_tests.rs
+++ b/codex-rs/protocol/src/local_media_tests.rs
@@ -31,8 +31,10 @@ fn snapshots_local_image_user_input_with_requested_detail() -> Result<()> {
assert_eq!(
UserInput::Image {
- image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGNgAAIAAAUAAXpeqz8AAAAASUVORK5CYII="
- .to_string(),
+ image: crate::models::ImageReference::Inline {
+ image_url: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGNgAAIAAAUAAXpeqz8AAAAASUVORK5CYII="
+ .to_string(),
+ },
detail,
},
input
diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs
index e2f588eaa3..549d69d553 100644
--- a/codex-rs/protocol/src/models.rs
+++ b/codex-rs/protocol/src/models.rs
@@ -898,6 +898,7 @@ pub enum ContentItem {
#[ts(untagged)]
pub enum ImageReference {
Inline { image_url: String },
+ File { file_id: String },
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema, TS)]
@@ -2008,13 +2009,11 @@ impl ResponseInputItem {
.into_iter()
.flat_map(|c| match c {
UserInput::Text { text, .. } => vec![ContentItem::InputText { text }],
- UserInput::Image {
- image_url, detail, ..
- } => {
+ UserInput::Image { image, detail, .. } => {
image_index += 1;
let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL);
vec![ContentItem::InputImage {
- image: ImageReference::Inline { image_url },
+ image,
detail: Some(detail),
}]
}
@@ -3885,7 +3884,9 @@ mod tests {
let image_url = "data:image/png;base64,abc".to_string();
let item = ResponseInputItem::from(vec![UserInput::Image {
- image_url: image_url.clone(),
+ image: ImageReference::Inline {
+ image_url: image_url.clone(),
+ },
detail: None,
}]);
@@ -3903,6 +3904,43 @@ mod tests {
Ok(())
}
+ /// A file-backed user image must reach the Responses API without being resolved by Core.
+ #[test]
+ fn file_image_user_input_serializes_file_id() -> Result<()> {
+ let file_id = "file_123".to_string();
+
+ let item = ResponseInputItem::from(vec![UserInput::Image {
+ image: ImageReference::File {
+ file_id: file_id.clone(),
+ },
+ detail: None,
+ }]);
+
+ let expected = ResponseInputItem::Message {
+ role: "user".to_string(),
+ content: vec![ContentItem::InputImage {
+ image: ImageReference::File { file_id },
+ detail: Some(DEFAULT_IMAGE_DETAIL),
+ }],
+ phase: None,
+ };
+ assert_eq!(item, expected);
+ assert_eq!(
+ serde_json::to_value(item)?,
+ serde_json::json!({
+ "type": "message",
+ "role": "user",
+ "content": [{
+ "type": "input_image",
+ "file_id": "file_123",
+ "detail": "high"
+ }]
+ })
+ );
+
+ Ok(())
+ }
+
#[test]
fn serializes_audio_user_input_without_tags() -> Result<()> {
let audio_url = "data:audio/wav;base64,abc".to_string();
@@ -4029,7 +4067,9 @@ mod tests {
let image_url = "data:image/png;base64,abc".to_string();
let item = ResponseInputItem::from(vec![UserInput::Image {
- image_url: image_url.clone(),
+ image: ImageReference::Inline {
+ image_url: image_url.clone(),
+ },
detail: Some(ImageDetail::Original),
}]);
@@ -4227,7 +4267,9 @@ mod tests {
let item = ResponseInputItem::from(vec![
UserInput::Image {
- image_url: image_url.clone(),
+ image: ImageReference::Inline {
+ image_url: image_url.clone(),
+ },
detail: None,
},
UserInput::LocalImage {
diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs
index ceeaa80e31..b08780eb49 100644
--- a/codex-rs/protocol/src/protocol.rs
+++ b/codex-rs/protocol/src/protocol.rs
@@ -2521,6 +2521,14 @@ pub struct AgentMessageEvent {
pub questions: Option<Vec<AsyncUserInputQuestion>>,
}
+#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)]
+#[serde(rename_all = "snake_case")]
+#[ts(rename_all = "snake_case")]
+pub enum UserMessageImageKind {
+ Inline,
+ File,
+}
+
#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq, JsonSchema, TS)]
pub struct UserMessageEvent {
#[serde(default, skip_serializing_if = "Option::is_none")]
@@ -2535,6 +2543,19 @@ pub struct UserMessageEvent {
/// default image detail behavior.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub image_details: Vec<Option<ImageDetail>>,
+ /// File IDs sourced from `UserInput::Image`. These are passed through as
+ /// opaque references and are not created by image preparation.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub file_ids: Option<Vec<String>>,
+ /// Detail hints for `file_ids`, indexed in parallel. Missing entries imply
+ /// default image detail behavior.
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
+ pub file_id_details: Vec<Option<ImageDetail>>,
+ /// Inline and file-backed image kinds in their original input order.
+ /// New producers populate this alongside `images` and `file_ids`; when it
+ /// is absent, consumers retain the legacy inline-then-file ordering.
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
+ pub image_order: Vec<UserMessageImageKind>,
/// Local file paths sourced from `UserInput::LocalImage`. These are kept so
/// the UI can reattach images when editing history. Local image prompts may
/// include a display form of the path, but these should not be treated as
@@ -2559,6 +2580,27 @@ pub struct UserMessageEvent {
pub text_elements: Vec<crate::user_input::TextElement>,
}
+impl UserMessageEvent {
+ /// Returns whether `image_order` accounts for every split image reference exactly once.
+ pub fn has_complete_image_order(&self) -> bool {
+ if self.image_order.is_empty() {
+ return false;
+ }
+
+ let mut inline_count = 0;
+ let mut file_count = 0;
+ for image_kind in &self.image_order {
+ match image_kind {
+ UserMessageImageKind::Inline => inline_count += 1,
+ UserMessageImageKind::File => file_count += 1,
+ }
+ }
+
+ inline_count == self.images.as_ref().map_or(0, Vec::len)
+ && file_count == self.file_ids.as_ref().map_or(0, Vec::len)
+ }
+}
+
/// Returns the user-facing preview text for a user message.
pub fn user_message_preview(user: &UserMessageEvent) -> Option<String> {
let message = strip_user_message_prefix(user.message.as_str());
@@ -2569,6 +2611,10 @@ pub fn user_message_preview(user: &UserMessageEvent) -> Option<String> {
.images
.as_ref()
.is_some_and(|images| !images.is_empty())
+ || user
+ .file_ids
+ .as_ref()
+ .is_some_and(|file_ids| !file_ids.is_empty())
|| !user.local_images.is_empty()
{
return Some("[Image]".to_string());
@@ -5944,6 +5990,9 @@ mod tests {
Some(vec!["https://example.com/image.png".to_string()])
);
assert_eq!(event.image_details, Vec::<Option<ImageDetail>>::new());
+ assert_eq!(event.file_ids, None);
+ assert_eq!(event.file_id_details, Vec::<Option<ImageDetail>>::new());
+ assert_eq!(event.image_order, Vec::<UserMessageImageKind>::new());
assert_eq!(event.local_images, vec![PathBuf::from("/tmp/local.png")]);
assert_eq!(event.local_image_details, Vec::<Option<ImageDetail>>::new());
assert_eq!(event.audio, None);
@@ -5959,11 +6008,21 @@ mod tests {
let local_audio_path = PathBuf::from("/tmp/local.wav");
let mut item = UserMessageItem::new(&[
crate::user_input::UserInput::Image {
- image_url: "https://example.com/first.png".to_string(),
+ image: crate::models::ImageReference::Inline {
+ image_url: "https://example.com/first.png".to_string(),
+ },
detail: Some(ImageDetail::Original),
},
crate::user_input::UserInput::Image {
- image_url: "https://example.com/second.png".to_string(),
+ image: crate::models::ImageReference::File {
+ file_id: "file_123".to_string(),
+ },
+ detail: Some(ImageDetail::Low),
+ },
+ crate::user_input::UserInput::Image {
+ image: crate::models::ImageReference::Inline {
+ image_url: "https://example.com/second.png".to_string(),
+ },
detail: None,
},
crate::user_input::UserInput::LocalImage {
@@ -5982,6 +6041,7 @@ mod tests {
let EventMsg::UserMessage(event) = item.as_legacy_event() else {
panic!("expected user message event");
};
+ let event_json = serde_json::to_value(&event).expect("serialize user message event");
assert_eq!(
event.images,
@@ -5992,6 +6052,22 @@ mod tests {
);
assert_eq!(event.client_id, Some("client-message-1".to_string()));
assert_eq!(event.image_details, vec![Some(ImageDetail::Original)]);
+ assert_eq!(event.file_ids, Some(vec!["file_123".to_string()]));
+ assert_eq!(event.file_id_details, vec![Some(ImageDetail::Low)]);
+ assert_eq!(
+ event.image_order,
+ vec![
+ UserMessageImageKind::Inline,
+ UserMessageImageKind::File,
+ UserMessageImageKind::Inline,
+ ]
+ );
+ assert_eq!(event_json["file_ids"], json!(["file_123"]));
+ assert_eq!(event_json["file_id_details"], json!(["low"]));
+ assert_eq!(
+ event_json["image_order"],
+ json!(["inline", "file", "inline"])
+ );
assert_eq!(event.local_images, vec![local_path]);
assert_eq!(event.local_image_details, vec![Some(ImageDetail::Original)]);
assert_eq!(
@@ -6011,6 +6087,16 @@ mod tests {
assert_eq!(user_message_preview(&event), Some("[Audio]".to_string()));
}
+ #[test]
+ fn file_only_user_message_has_placeholder_preview() {
+ let event = UserMessageEvent {
+ file_ids: Some(vec!["file_123".to_string()]),
+ ..Default::default()
+ };
+
+ assert_eq!(user_message_preview(&event), Some("[Image]".to_string()));
+ }
+
#[test]
fn turn_aborted_event_deserializes_without_turn_id() -> Result<()> {
let event: EventMsg = serde_json::from_value(json!({
diff --git a/codex-rs/protocol/src/user_input.rs b/codex-rs/protocol/src/user_input.rs
index 9cb27e897d..8f5f87c00a 100644
--- a/codex-rs/protocol/src/user_input.rs
+++ b/codex-rs/protocol/src/user_input.rs
@@ -4,6 +4,7 @@ use serde::Serialize;
use ts_rs::TS;
use crate::models::ImageDetail;
+use crate::models::ImageReference;
/// Conservative cap so one user message cannot monopolize a large context window.
pub const MAX_USER_INPUT_TEXT_CHARS: usize = 1 << 20;
@@ -22,9 +23,10 @@ pub enum UserInput {
#[serde(default)]
text_elements: Vec<TextElement>,
},
- /// Pre‑encoded data: URI image.
+ /// Image reference forwarded to the Responses API.
Image {
- image_url: String,
+ #[serde(flatten)]
+ image: ImageReference,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional)]
detail: Option<ImageDetail>,
diff --git a/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs b/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs
index 589509f7ff..734327261a 100644
--- a/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs
+++ b/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs
@@ -26,8 +26,10 @@ use codex_protocol::items::SubAgentActivityItem;
use codex_protocol::items::TurnItem;
use codex_protocol::items::UserMessageItem;
use codex_protocol::items::WebSearchItem;
+use codex_protocol::models::ImageReference;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::UserMessageEvent;
+use codex_protocol::protocol::UserMessageImageKind;
use codex_protocol::user_input::UserInput;
use crate::ThreadStoreResult;
@@ -37,20 +39,71 @@ pub(super) fn user_message_item(
next_item_id: &mut impl FnMut() -> ThreadStoreResult<String>,
) -> ThreadStoreResult<TurnItem> {
let mut content = Vec::new();
+ let has_image_order = event.has_complete_image_order();
if !event.message.trim().is_empty() {
content.push(UserInput::Text {
text: event.message,
text_elements: event.text_elements,
});
}
- if let Some(images) = event.images {
+ if has_image_order {
+ let mut inline_index = 0;
+ let mut file_index = 0;
+ for image_kind in event.image_order {
+ let (image, detail) = match image_kind {
+ UserMessageImageKind::Inline => {
+ let Some(image_url) = event
+ .images
+ .as_deref()
+ .and_then(|images| images.get(inline_index))
+ else {
+ continue;
+ };
+ let detail = event.image_details.get(inline_index).copied().flatten();
+ inline_index += 1;
+ (
+ ImageReference::Inline {
+ image_url: image_url.clone(),
+ },
+ detail,
+ )
+ }
+ UserMessageImageKind::File => {
+ let Some(file_id) = event
+ .file_ids
+ .as_deref()
+ .and_then(|file_ids| file_ids.get(file_index))
+ else {
+ continue;
+ };
+ let detail = event.file_id_details.get(file_index).copied().flatten();
+ file_index += 1;
+ (
+ ImageReference::File {
+ file_id: file_id.clone(),
+ },
+ detail,
+ )
+ }
+ };
+ content.push(UserInput::Image { image, detail });
+ }
+ } else if let Some(images) = event.images {
for (index, image_url) in images.into_iter().enumerate() {
content.push(UserInput::Image {
- image_url,
+ image: ImageReference::Inline { image_url },
detail: event.image_details.get(index).copied().flatten(),
});
}
}
+ if !has_image_order && let Some(file_ids) = event.file_ids {
+ for (index, file_id) in file_ids.into_iter().enumerate() {
+ content.push(UserInput::Image {
+ image: ImageReference::File { file_id },
+ detail: event.file_id_details.get(index).copied().flatten(),
+ });
+ }
+ }
for (index, path) in event.local_images.into_iter().enumerate() {
content.push(UserInput::LocalImage {
path,
diff --git a/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs b/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs
index 07da7b7042..11a8c8d6d6 100644
--- a/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs
+++ b/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs
@@ -16,6 +16,7 @@ use codex_protocol::models::ImageReference;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::UserMessageEvent;
+use codex_protocol::protocol::UserMessageImageKind;
use codex_rollout::CompactedItem;
use codex_rollout::RetainedContextEntry;
use codex_rollout::RetainedInputSource;
@@ -489,19 +490,28 @@ fn explicit_event_turn_id(event: &EventMsg) -> Option<&str> {
fn user_response_matches_event(content: &[ContentItem], event: &UserMessageEvent) -> bool {
let mut text = String::new();
let mut images = Vec::new();
+ let mut file_ids = Vec::new();
+ let mut image_order = Vec::new();
let mut audio = Vec::new();
for item in content {
match item {
ContentItem::InputText { text: item_text } => text.push_str(item_text),
- ContentItem::InputImage {
- image: ImageReference::Inline { image_url },
- ..
- } => images.push(image_url.as_str()),
+ ContentItem::InputImage { image, .. } => match image {
+ ImageReference::Inline { image_url } => {
+ image_order.push(UserMessageImageKind::Inline);
+ images.push(image_url.as_str());
+ }
+ ImageReference::File { file_id } => {
+ image_order.push(UserMessageImageKind::File);
+ file_ids.push(file_id.as_str());
+ }
+ },
ContentItem::InputAudio { audio_url } => audio.push(audio_url.as_str()),
ContentItem::OutputText { .. } => return false,
}
}
text == event.message
+ && (!event.has_complete_image_order() || image_order == event.image_order)
&& images
== event
.images
@@ -510,6 +520,14 @@ fn user_response_matches_event(content: &[ContentItem], event: &UserMessageEvent
.iter()
.map(String::as_str)
.collect::<Vec<_>>()
+ && file_ids
+ == event
+ .file_ids
+ .as_deref()
+ .unwrap_or_default()
+ .iter()
+ .map(String::as_str)
+ .collect::<Vec<_>>()
&& audio
== event
.audio
diff --git a/codex-rs/thread-store/src/local/rollout_migration_tests.rs b/codex-rs/thread-store/src/local/rollout_migration_tests.rs
index bad11829c6..7c85b6b080 100644
--- a/codex-rs/thread-store/src/local/rollout_migration_tests.rs
+++ b/codex-rs/thread-store/src/local/rollout_migration_tests.rs
@@ -20,6 +20,8 @@ use codex_protocol::mcp::McpResourceOrigin;
use codex_protocol::mcp::McpResourceOriginCheckpoint;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ContentItemKind;
+use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::InternalChatMessageMetadataPassthrough;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::AgentMessageEvent;
@@ -39,6 +41,8 @@ use codex_protocol::protocol::TurnCompleteEvent;
use codex_protocol::protocol::TurnContextItem;
use codex_protocol::protocol::TurnStartedEvent;
use codex_protocol::protocol::UserMessageEvent;
+use codex_protocol::protocol::UserMessageImageKind;
+use codex_protocol::user_input::UserInput;
use codex_rollout::CompactedItem;
use codex_rollout::RolloutConfig;
use codex_rollout::RolloutItem;
@@ -318,12 +322,21 @@ async fn list_active_summary_turns(store: &LocalThreadStore, thread_id: ThreadId
async fn migration_publishes_canonical_projected_history_and_is_idempotent() {
let home = TempDir::new().expect("create Codex home");
let thread_id = ThreadId::new();
+ let user_event = UserMessageEvent {
+ message: "first question".to_string(),
+ images: Some(vec!["https://example.com/image.png".to_string()]),
+ image_details: vec![Some(ImageDetail::Original)],
+ file_ids: Some(vec!["file_123".to_string()]),
+ file_id_details: vec![Some(ImageDetail::High)],
+ image_order: vec![UserMessageImageKind::File, UserMessageImageKind::Inline],
+ ..Default::default()
+ };
let path = write_rollout(
home.path(),
thread_id,
SessionSource::Cli,
vec![
- user_message("first question"),
+ RolloutItem::EventMsg(EventMsg::UserMessage(user_event)),
agent_message("first answer"),
],
);
@@ -355,6 +368,32 @@ async fn migration_publishes_canonical_projected_history_and_is_idempotent() {
.count(),
2
);
+ let user_item = lines.iter().find_map(|line| match &line.item {
+ RolloutItem::EventMsg(EventMsg::ItemCompleted(ItemCompletedEvent {
+ item: TurnItem::UserMessage(item),
+ ..
+ })) => Some(item),
+ _ => None,
+ });
+ let expected_content = vec![
+ UserInput::Text {
+ text: "first question".to_string(),
+ text_elements: Vec::new(),
+ },
+ UserInput::Image {
+ image: ImageReference::File {
+ file_id: "file_123".to_string(),
+ },
+ detail: Some(ImageDetail::High),
+ },
+ UserInput::Image {
+ image: ImageReference::Inline {
+ image_url: "https://example.com/image.png".to_string(),
+ },
+ detail: Some(ImageDetail::Original),
+ },
+ ];
+ assert_eq!(user_item.map(|item| &item.content), Some(&expected_content));
let turns = list_active_summary_turns(&store, thread_id).await;
assert_eq!(turns.turns.len(), 1);
@@ -847,13 +886,36 @@ async fn migration_drops_trailing_context_when_rollback_arrives_before_next_turn
async fn migration_coalesces_response_first_user_message_rollback_boundary() {
let home = TempDir::new().expect("create Codex home");
let thread_id = ThreadId::new();
+ let file_id = "file_123".to_string();
+ let response = ResponseItem::Message {
+ id: None,
+ role: "user".to_string(),
+ content: vec![
+ ContentItem::InputText {
+ text: "remove question".to_string(),
+ },
+ ContentItem::InputImage {
+ image: ImageReference::File {
+ file_id: file_id.clone(),
+ },
+ detail: None,
+ },
+ ],
+ phase: None,
+ internal_chat_message_metadata_passthrough: None,
+ };
+ let event = UserMessageEvent {
+ message: "remove question".to_string(),
+ file_ids: Some(vec![file_id]),
+ ..Default::default()
+ };
let path = write_rollout(
home.path(),
thread_id,
SessionSource::Cli,
vec![
- rollout_response_item(input_response_message("user", "remove question")),
- user_message("remove question"),
+ rollout_response_item(response),
+ RolloutItem::EventMsg(EventMsg::UserMessage(event)),
RolloutItem::EventMsg(EventMsg::ThreadRolledBack(ThreadRolledBackEvent {
num_turns: 1,
})),
diff --git a/codex-rs/tools/src/tool_output.rs b/codex-rs/tools/src/tool_output.rs
index a299d172ee..b4cf87be73 100644
--- a/codex-rs/tools/src/tool_output.rs
+++ b/codex-rs/tools/src/tool_output.rs
@@ -220,13 +220,12 @@ fn response_input_to_code_mode_result(response: ResponseInputItem) -> JsonValue
| codex_protocol::models::ContentItem::OutputText { text } => {
FunctionCallOutputContentItem::InputText { text }
}
- codex_protocol::models::ContentItem::InputImage {
- image: ImageReference::Inline { image_url },
- detail,
- } => FunctionCallOutputContentItem::InputImage {
- image: ImageReference::Inline { image_url },
- detail: detail.or(Some(DEFAULT_IMAGE_DETAIL)),
- },
+ codex_protocol::models::ContentItem::InputImage { image, detail } => {
+ FunctionCallOutputContentItem::InputImage {
+ image,
+ detail: detail.or(Some(DEFAULT_IMAGE_DETAIL)),
+ }
+ }
codex_protocol::models::ContentItem::InputAudio { audio_url } => {
FunctionCallOutputContentItem::InputAudio { audio_url }
}
@@ -260,6 +259,10 @@ fn content_items_to_code_mode_result(items: &[FunctionCallOutputContentItem]) ->
image: ImageReference::Inline { image_url },
..
} if !image_url.trim().is_empty() => Some(image_url.clone()),
+ FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::File { file_id },
+ ..
+ } if !file_id.trim().is_empty() => Some(file_id.clone()),
FunctionCallOutputContentItem::InputAudio { audio_url }
if !audio_url.trim().is_empty() =>
{
@@ -274,3 +277,7 @@ fn content_items_to_code_mode_result(items: &[FunctionCallOutputContentItem]) ->
.join("\n"),
)
}
+
+#[cfg(test)]
+#[path = "tool_output_tests.rs"]
+mod tests;
diff --git a/codex-rs/tools/src/tool_output_tests.rs b/codex-rs/tools/src/tool_output_tests.rs
new file mode 100644
index 0000000000..263603e99b
--- /dev/null
+++ b/codex-rs/tools/src/tool_output_tests.rs
@@ -0,0 +1,25 @@
+use codex_protocol::models::ContentItem;
+use codex_protocol::models::ImageReference;
+use codex_protocol::models::ResponseInputItem;
+use pretty_assertions::assert_eq;
+
+use super::response_input_to_code_mode_result;
+
+#[test]
+fn code_mode_text_preserves_file_backed_image_id() {
+ let response = ResponseInputItem::Message {
+ role: "user".to_string(),
+ content: vec![ContentItem::InputImage {
+ image: ImageReference::File {
+ file_id: "file_123".to_string(),
+ },
+ detail: None,
+ }],
+ phase: None,
+ };
+
+ assert_eq!(
+ response_input_to_code_mode_result(response),
+ serde_json::json!("file_123")
+ );
+}
diff --git a/codex-rs/tui/src/chatwidget/image_submission.rs b/codex-rs/tui/src/chatwidget/image_submission.rs
index 3e5a961158..832104c555 100644
--- a/codex-rs/tui/src/chatwidget/image_submission.rs
+++ b/codex-rs/tui/src/chatwidget/image_submission.rs
@@ -4,6 +4,7 @@
use super::*;
use codex_protocol::models::ImageDetail;
+use codex_protocol::models::ImageReference;
use codex_protocol::models::snapshot_local_user_input;
use codex_protocol::user_input::UserInput as CoreUserInput;
use tokio::sync::oneshot;
@@ -149,7 +150,11 @@ fn prepare_images(
return Err(too_large());
}
snapshot_local_user_input(&mut input)?;
- if let CoreUserInput::Image { image_url, .. } = &input {
+ if let CoreUserInput::Image {
+ image: ImageReference::Inline { image_url },
+ ..
+ } = &input
+ {
remaining_image_bytes = remaining_image_bytes
.checked_sub(image_url.len())
.ok_or_else(too_large)?;
diff --git a/codex-rs/tui/src/chatwidget/input_submission.rs b/codex-rs/tui/src/chatwidget/input_submission.rs
index 6b43d3793e..d4567b6be9 100644
--- a/codex-rs/tui/src/chatwidget/input_submission.rs
+++ b/codex-rs/tui/src/chatwidget/input_submission.rs
@@ -1,6 +1,7 @@
//! User-message and shell-prompt submission behavior for `ChatWidget`.
use super::*;
+use codex_app_server_protocol::ImageReference;
impl ChatWidget {
pub(crate) fn set_task_mentions_enabled(&mut self, enabled: bool) {
@@ -239,7 +240,9 @@ impl ChatWidget {
for image_url in &remote_image_urls {
items.push(UserInput::Image {
- url: image_url.clone(),
+ image: ImageReference::Inline {
+ url: image_url.clone(),
+ },
detail: None,
});
}
diff --git a/codex-rs/tui/src/chatwidget/tests/composer_submission.rs b/codex-rs/tui/src/chatwidget/tests/composer_submission.rs
index e8deed5e9e..1d1db72362 100644
--- a/codex-rs/tui/src/chatwidget/tests/composer_submission.rs
+++ b/codex-rs/tui/src/chatwidget/tests/composer_submission.rs
@@ -2,6 +2,7 @@ use super::*;
use crate::app_event::ConnectorsSnapshot;
use crate::history_cell::ThreadRecapLoadingCell;
use base64::Engine;
+use codex_app_server_protocol::ImageReference;
use codex_protocol::models::ManagedFileSystemPermissions;
use codex_protocol::permissions::FileSystemAccessMode;
use codex_protocol::permissions::FileSystemPath;
@@ -640,7 +641,9 @@ async fn submission_with_remote_and_local_images_keeps_local_placeholder_numberi
assert_eq!(
items[0],
UserInput::Image {
- url: remote_url.clone(),
+ image: ImageReference::Inline {
+ url: remote_url.clone(),
+ },
detail: None,
}
);
@@ -727,7 +730,9 @@ async fn enter_with_only_remote_images_submits_user_turn() {
assert_eq!(
items,
vec![UserInput::Image {
- url: remote_url.clone(),
+ image: ImageReference::Inline {
+ url: remote_url.clone(),
+ },
detail: None,
}]
);
@@ -2157,7 +2162,9 @@ fn user_message_display_from_inputs_matches_flattened_user_message_shape() {
text_elements: vec![TextElement::new((0..5).into(), /*placeholder*/ None).into()],
},
UserInput::Image {
- url: "https://example.com/remote.png".to_string(),
+ image: ImageReference::Inline {
+ url: "https://example.com/remote.png".to_string(),
+ },
detail: None,
},
UserInput::LocalImage {
@@ -2522,7 +2529,13 @@ async fn image_submission_is_portable_for_new_turns_and_steers() {
_ => {}
}
};
- let [UserInput::Image { url, detail }, UserInput::Text { .. }] = items.as_slice()
+ let [
+ UserInput::Image {
+ image: ImageReference::Inline { url },
+ detail,
+ },
+ UserInput::Text { .. },
+ ] = items.as_slice()
else {
panic!("local image must be portable on the wire");
};
diff --git a/codex-rs/tui/src/chatwidget/tests/history_replay.rs b/codex-rs/tui/src/chatwidget/tests/history_replay.rs
index fbc15c4ec4..5979f32662 100644
--- a/codex-rs/tui/src/chatwidget/tests/history_replay.rs
+++ b/codex-rs/tui/src/chatwidget/tests/history_replay.rs
@@ -1,5 +1,6 @@
use super::*;
use crate::app_event::HistoryLookupResponse;
+use codex_app_server_protocol::ImageReference;
use codex_app_server_protocol::NetworkAccess;
use codex_app_server_protocol::SandboxPolicy;
use codex_protocol::models::FunctionCallOutputBody;
@@ -523,7 +524,9 @@ async fn replayed_user_message_preserves_remote_image_urls() {
text_elements: Vec::new(),
},
AppServerUserInput::Image {
- url: remote_image_urls[0].clone(),
+ image: ImageReference::Inline {
+ url: remote_image_urls[0].clone(),
+ },
detail: None,
},
],
@@ -783,7 +786,9 @@ async fn replayed_user_message_with_only_remote_images_renders_history_cell() {
&mut chat,
"user-1",
vec![AppServerUserInput::Image {
- url: remote_image_urls[0].clone(),
+ image: ImageReference::Inline {
+ url: remote_image_urls[0].clone(),
+ },
detail: None,
}],
ReplayKind::ResumeInitialMessages,
diff --git a/codex-rs/tui/src/chatwidget/tests/review_mode.rs b/codex-rs/tui/src/chatwidget/tests/review_mode.rs
index f947b364d2..7107273e6c 100644
--- a/codex-rs/tui/src/chatwidget/tests/review_mode.rs
+++ b/codex-rs/tui/src/chatwidget/tests/review_mode.rs
@@ -1,4 +1,5 @@
use super::*;
+use codex_app_server_protocol::ImageReference;
use pretty_assertions::assert_eq;
use std::collections::VecDeque;
@@ -646,7 +647,9 @@ async fn item_completed_pops_pending_steer_with_local_image_and_text_elements()
"user-1",
vec![
UserInput::Image {
- url: "data:image/png;base64,placeholder".to_string(),
+ image: ImageReference::Inline {
+ url: "data:image/png;base64,placeholder".to_string(),
+ },
detail: None,
},
UserInput::Text {
diff --git a/codex-rs/tui/src/chatwidget/user_messages.rs b/codex-rs/tui/src/chatwidget/user_messages.rs
index 59dec1f6d2..befc76f54a 100644
--- a/codex-rs/tui/src/chatwidget/user_messages.rs
+++ b/codex-rs/tui/src/chatwidget/user_messages.rs
@@ -14,6 +14,7 @@ use std::path::PathBuf;
use crate::bottom_pane::LocalImageAttachment;
use crate::bottom_pane::MentionBinding;
use crate::bottom_pane::QueuedInputAction;
+use codex_app_server_protocol::ImageReference;
use codex_app_server_protocol::TextElement as AppServerTextElement;
use codex_app_server_protocol::UserInput;
use codex_protocol::config_types::CollaborationMode;
@@ -739,6 +740,20 @@ impl ChatWidget {
{
tracing::warn!("audio user inputs are not supported by the TUI and will be omitted");
}
+ // TODO(kc) preserve file-backed images when the TUI can resolve or replay them.
+ if items.iter().any(|item| {
+ matches!(
+ item,
+ UserInput::Image {
+ image: ImageReference::File { .. },
+ ..
+ }
+ )
+ }) {
+ tracing::warn!(
+ "file-backed image inputs are not supported by the TUI and will be omitted"
+ );
+ }
let mut message = String::new();
let mut remote_image_urls = Vec::new();
let mut local_images = Vec::new();
@@ -765,7 +780,14 @@ impl ChatWidget {
)
}),
),
- UserInput::Image { url, .. } => remote_image_urls.push(url.clone()),
+ UserInput::Image {
+ image: ImageReference::Inline { url },
+ ..
+ } => remote_image_urls.push(url.clone()),
+ UserInput::Image {
+ image: ImageReference::File { .. },
+ ..
+ } => {}
UserInput::LocalImage { path, .. } => local_images.push(path.clone()),
UserInput::Audio { .. } // TODO: Include audio inputs in the user message display.
| UserInput::LocalAudio { .. } // TODO: Include audio inputs in the user message display.
diff --git a/codex-rs/tui/src/dynamic_tools.rs b/codex-rs/tui/src/dynamic_tools.rs
index 8633fdaf58..e432195d20 100644
--- a/codex-rs/tui/src/dynamic_tools.rs
+++ b/codex-rs/tui/src/dynamic_tools.rs
@@ -12,6 +12,7 @@ use codex_app_server_protocol::DynamicToolFunctionSpec;
use codex_app_server_protocol::DynamicToolNamespaceSpec;
use codex_app_server_protocol::DynamicToolNamespaceTool;
use codex_app_server_protocol::DynamicToolSpec;
+use codex_app_server_protocol::ImageReference;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::SandboxMode;
use codex_app_server_protocol::SandboxPolicy;
@@ -1340,7 +1341,14 @@ fn turn_summary(turn: &Turn, include_outputs: bool, output_chars: usize) -> Valu
}
input
}
- UserInput::Image { url, .. } => json!({"type": "image", "url": url}),
+ UserInput::Image {
+ image: ImageReference::Inline { url },
+ ..
+ } => json!({"type": "image", "url": url}),
+ UserInput::Image {
+ image: ImageReference::File { file_id },
+ ..
+ } => json!({"type": "image", "fileId": file_id}),
UserInput::LocalImage { path, .. } => json!({"type": "localImage", "path": path}),
UserInput::Audio { url } => json!({"type": "audio", "url": url}),
UserInput::LocalAudio { path } => json!({"type": "localAudio", "path": path}),
diff --git a/codex-rs/tui/src/resume_picker_transcript_preview_tests.rs b/codex-rs/tui/src/resume_picker_transcript_preview_tests.rs
index 0c406185e9..401d7ad205 100644
--- a/codex-rs/tui/src/resume_picker_transcript_preview_tests.rs
+++ b/codex-rs/tui/src/resume_picker_transcript_preview_tests.rs
@@ -4,6 +4,7 @@ use app_test_support::create_mock_responses_server_sequence;
use app_test_support::write_mock_responses_config_toml;
use codex_app_server_client::AppServerEvent;
use codex_app_server_protocol::ClientRequest;
+use codex_app_server_protocol::ImageReference;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::ServerNotification;
use codex_app_server_protocol::SortDirection;
@@ -362,9 +363,11 @@ async fn transcript_preview_for_history_mode(
}
} else {
UserInput::Image {
- url: String::from(
- "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==",
- ),
+ image: ImageReference::Inline {
+ url: String::from(
+ "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==",
+ ),
+ },
detail: None,
}
};
diff --git a/codex-rs/utils/output-truncation/src/lib.rs b/codex-rs/utils/output-truncation/src/lib.rs
index da59e7b86d..1cae07741c 100644
--- a/codex-rs/utils/output-truncation/src/lib.rs
+++ b/codex-rs/utils/output-truncation/src/lib.rs
@@ -3,7 +3,6 @@
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
-use codex_protocol::models::ImageReference;
pub use codex_utils_string::approx_bytes_for_tokens;
pub use codex_utils_string::approx_token_count;
pub use codex_utils_string::approx_tokens_from_byte_count;
@@ -91,15 +90,12 @@ pub fn formatted_truncate_text_content_items_with_policy(
text: formatted_truncate_text(&combined, policy),
}];
out.extend(items.iter().filter_map(|item| match item {
- FunctionCallOutputContentItem::InputImage {
- image: ImageReference::Inline { image_url },
- detail,
- } => Some(FunctionCallOutputContentItem::InputImage {
- image: ImageReference::Inline {
- image_url: image_url.clone(),
- },
- detail: *detail,
- }),
+ FunctionCallOutputContentItem::InputImage { image, detail } => {
+ Some(FunctionCallOutputContentItem::InputImage {
+ image: image.clone(),
+ detail: *detail,
+ })
+ }
FunctionCallOutputContentItem::InputAudio { audio_url } => {
Some(FunctionCallOutputContentItem::InputAudio {
audio_url: audio_url.clone(),
@@ -163,14 +159,9 @@ pub fn truncate_function_output_items_with_policy(
remaining_budget = 0;
}
}
- FunctionCallOutputContentItem::InputImage {
- image: ImageReference::Inline { image_url },
- detail,
- } => {
+ FunctionCallOutputContentItem::InputImage { image, detail } => {
out.push(FunctionCallOutputContentItem::InputImage {
- image: ImageReference::Inline {
- image_url: image_url.clone(),
- },
+ image: image.clone(),
detail: *detail,
});
}
diff --git a/codex-rs/utils/output-truncation/src/truncate_tests.rs b/codex-rs/utils/output-truncation/src/truncate_tests.rs
index bbcf617714..736a283563 100644
--- a/codex-rs/utils/output-truncation/src/truncate_tests.rs
+++ b/codex-rs/utils/output-truncation/src/truncate_tests.rs
@@ -193,6 +193,12 @@ fn truncate_function_output_items_with_policy_discards_empty_text() {
},
detail: Some(DEFAULT_IMAGE_DETAIL),
},
+ FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::File {
+ file_id: "file_123".to_string(),
+ },
+ detail: Some(DEFAULT_IMAGE_DETAIL),
+ },
FunctionCallOutputContentItem::InputAudio {
audio_url: "audio:one".to_string(),
},
@@ -271,6 +277,12 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_med
FunctionCallOutputContentItem::InputAudio {
audio_url: "audio:one".to_string(),
},
+ FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::File {
+ file_id: "file_123".to_string(),
+ },
+ detail: Some(DEFAULT_IMAGE_DETAIL),
+ },
FunctionCallOutputContentItem::InputText {
text: "ijkl".to_string(),
},
@@ -298,6 +310,12 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_med
FunctionCallOutputContentItem::InputAudio {
audio_url: "audio:one".to_string(),
},
+ FunctionCallOutputContentItem::InputImage {
+ image: ImageReference::File {
+ file_id: "file_123".to_string(),
+ },
+ detail: Some(DEFAULT_IMAGE_DETAIL),
+ },
FunctionCallOutputContentItem::InputImage {
image: ImageReference::Inline { image_url: "img:two".to_string() },
detail: Some(DEFAULT_IMAGE_DETAIL),
diff --git a/sdk/python/src/openai_codex/generated/v2_all.py b/sdk/python/src/openai_codex/generated/v2_all.py
index 5b8e03a633..0a76ae8fc4 100644
--- a/sdk/python/src/openai_codex/generated/v2_all.py
+++ b/sdk/python/src/openai_codex/generated/v2_all.py
@@ -6240,7 +6240,7 @@ class TextUserInput(BaseModel):
type: Annotated[Literal["text"], Field(title="TextUserInputType")]
-class ImageUserInput(BaseModel):
+class UrlUserInput(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
@@ -6249,6 +6249,15 @@ class ImageUserInput(BaseModel):
url: str
+class FileIdUserInput(BaseModel):
+ model_config = ConfigDict(
+ populate_by_name=True,
+ )
+ detail: ImageDetail | None = None
+ type: Annotated[Literal["image"], Field(title="ImageUserInputType")]
+ file_id: Annotated[str, Field(alias="fileId")]
+
+
class LocalImageUserInput(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
@@ -6295,7 +6304,8 @@ class MentionUserInput(BaseModel):
class UserInput(
RootModel[
TextUserInput
- | ImageUserInput
+ | UrlUserInput
+ | FileIdUserInput
| LocalImageUserInput
| AudioUserInput
| LocalAudioUserInput
@@ -6308,7 +6318,8 @@ class UserInput(
)
root: (
TextUserInput
- | ImageUserInput
+ | UrlUserInput
+ | FileIdUserInput
| LocalImageUserInput
| AudioUserInput
| LocalAudioUserInput
@@ -7794,15 +7805,34 @@ class InputImageContentItem(BaseModel):
image_url: str
+class FileIdContentItem(BaseModel):
+ model_config = ConfigDict(
+ populate_by_name=True,
+ )
+ detail: ImageDetail | None = None
+ type: Annotated[Literal["input_image"], Field(title="InputImageContentItemType")]
+ file_id: str
+
+
class ContentItem(
RootModel[
- InputTextContentItem | InputImageContentItem | InputAudioContentItem | OutputTextContentItem
+ InputTextContentItem
+ | InputImageContentItem
+ | FileIdContentItem
+ | InputAudioContentItem
+ | OutputTextContentItem
]
):
model_config = ConfigDict(
populate_by_name=True,
)
- root: InputTextContentItem | InputImageContentItem | InputAudioContentItem | OutputTextContentItem
+ root: (
+ InputTextContentItem
+ | InputImageContentItem
+ | FileIdContentItem
+ | InputAudioContentItem
+ | OutputTextContentItem
+ )
class ExperimentalFeature(BaseModel):
@@ -7990,10 +8020,22 @@ class InputImageFunctionCallOutputContentItem(BaseModel):
image_url: str
+class FileIdFunctionCallOutputContentItem(BaseModel):
+ model_config = ConfigDict(
+ populate_by_name=True,
+ )
+ detail: ImageDetail | None = None
+ type: Annotated[
+ Literal["input_image"], Field(title="InputImageFunctionCallOutputContentItemType")
+ ]
+ file_id: str
+
+
class FunctionCallOutputContentItem(
RootModel[
InputTextFunctionCallOutputContentItem
| InputImageFunctionCallOutputContentItem
+ | FileIdFunctionCallOutputContentItem
| InputAudioFunctionCallOutputContentItem
| EncryptedContentFunctionCallOutputContentItem
]
@@ -8004,6 +8046,7 @@ class FunctionCallOutputContentItem(
root: Annotated[
InputTextFunctionCallOutputContentItem
| InputImageFunctionCallOutputContentItem
+ | FileIdFunctionCallOutputContentItem
| InputAudioFunctionCallOutputContentItem
| EncryptedContentFunctionCallOutputContentItem,
Field(