Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Add streaming file replacement to filesystem abstractions (#54505)

Adam Perry @ OpenAIa45b8ccb016f
diff --git a/codex-rs/exec-server/src/client.rs b/codex-rs/exec-server/src/client.rs
index 2ce504dabb..6f1f74114b 100644
--- a/codex-rs/exec-server/src/client.rs
+++ b/codex-rs/exec-server/src/client.rs
@@ -1,5 +1,6 @@
use std::collections::BTreeMap;
use std::collections::HashMap;
+use std::future::Future;
use std::sync::Arc;
use std::sync::Mutex as StdMutex;
use std::sync::OnceLock;
@@ -969,21 +970,32 @@ impl ExecServerClient {
.await
}
- pub async fn fs_open(&self, params: FsOpenParams) -> Result<FsOpenResponse, ExecServerError> {
- // Older executors ignore the mode field and could silently return a read-only handle.
- if params.mode == FsOpenMode::Replace
- && !self
- .environment_info()
- .await?
- .capabilities
- .file_write_streaming
- {
- return Err(ExecServerError::Protocol(
- "exec-server does not support writable file streams".to_string(),
- ));
+ pub(crate) async fn supports_file_write_streaming(&self) -> Result<bool, ExecServerError> {
+ Ok(self
+ .environment_info()
+ .await?
+ .capabilities
+ .file_write_streaming)
+ }
+
+ /// Returns an owned future so callers can finish an open after they stop waiting for it.
+ pub fn fs_open(
+ &self,
+ params: FsOpenParams,
+ ) -> impl Future<Output = Result<FsOpenResponse, ExecServerError>> + Send + 'static {
+ let client = self.clone();
+ async move {
+ // Older executors ignore the mode field and could silently return a read-only handle.
+ if params.mode == FsOpenMode::Replace && !client.supports_file_write_streaming().await?
+ {
+ return Err(ExecServerError::Protocol(
+ "exec-server does not support writable file streams".to_string(),
+ ));
+ }
+ client
+ .call(FS_OPEN_METHOD, &WireFsOpenParams::from(params))
+ .await
}
- self.call(FS_OPEN_METHOD, &WireFsOpenParams::from(params))
- .await
}
pub async fn fs_read_block(
@@ -997,12 +1009,7 @@ impl ExecServerClient {
&self,
params: FsWriteBlockParams,
) -> Result<FsWriteBlockResponse, ExecServerError> {
- if !self
- .environment_info()
- .await?
- .capabilities
- .file_write_streaming
- {
+ if !self.supports_file_write_streaming().await? {
return Err(ExecServerError::Protocol(
"exec-server does not support writable file streams".to_string(),
));
@@ -1958,6 +1965,7 @@ mod tests {
use codex_exec_server_protocol::JSONRPCResponse;
use codex_http_client::HttpClientFactory;
use codex_http_client::OutboundProxyPolicy;
+ use codex_protocol::models::PermissionProfile;
use codex_utils_path_uri::PathUri;
use futures::SinkExt;
use futures::StreamExt;
@@ -1996,8 +2004,12 @@ mod tests {
use super::ExecServerClientConnectOptions;
use super::ExecServerError;
use super::LazyRemoteExecServerClient;
+ use crate::Environment;
use crate::EnvironmentObservedStatus;
+ use crate::ExecutorFileSystem;
+ use crate::FileSystemSandboxContext;
use crate::ProcessId;
+ use crate::client_api::DEFAULT_REMOTE_EXEC_SERVER_CONNECT_TIMEOUT;
#[cfg(not(windows))]
use crate::client_api::DEFAULT_REMOTE_EXEC_SERVER_INITIALIZE_TIMEOUT;
use crate::client_api::ExecServerTransportParams;
@@ -2019,19 +2031,24 @@ mod tests {
use crate::protocol::ExecOutputStream;
use crate::protocol::ExecParams;
use crate::protocol::ExecResponse;
+ use crate::protocol::FS_CLOSE_METHOD;
use crate::protocol::FS_OPEN_METHOD;
+ use crate::protocol::FS_WRITE_FILE_METHOD;
use crate::protocol::FsOpenMode;
use crate::protocol::FsOpenParams;
use crate::protocol::FsWriteBlockParams;
+ use crate::protocol::FsWriteFileParams;
use crate::protocol::INITIALIZE_METHOD;
use crate::protocol::INITIALIZED_METHOD;
use crate::protocol::InitializeResponse;
use crate::protocol::ProcessOutputChunk;
use crate::protocol::ProcessSandboxType;
use crate::protocol::ReadResponse;
+ use crate::protocol::WireFsWriteFileParams;
use crate::protocol::WriteParams;
use crate::protocol::WriteResponse;
use crate::protocol::WriteStatus;
+ use crate::remote_file_system::RemoteFileSystem;
async fn read_jsonrpc_line<R>(lines: &mut tokio::io::Lines<BufReader<R>>) -> JSONRPCMessage
where
@@ -2758,9 +2775,10 @@ mod tests {
Ok(())
}
- /// Old executors must never receive writable opens they would silently treat as reads.
+ /// Old executors receive one sandboxed writeFile, never an open they would mistake for a read.
#[tokio::test]
- async fn writable_file_streams_require_executor_capability() -> anyhow::Result<()> {
+ async fn old_executor_receives_buffered_replacement_and_keeps_read_only_opens()
+ -> anyhow::Result<()> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let websocket_url = format!("ws://{}", listener.local_addr()?);
let read_open = FsOpenParams {
@@ -2769,6 +2787,16 @@ mod tests {
mode: FsOpenMode::Read,
sandbox: None,
};
+ let sandbox = FileSystemSandboxContext::from_permission_profile(
+ PermissionProfile::workspace_write(),
+ PathUri::parse("file:///tmp")?,
+ );
+ let expected_write = WireFsWriteFileParams::from(FsWriteFileParams {
+ path: read_open.path.clone(),
+ data_base64: "AP9oaQ==".to_string(),
+ follow_symlinks: None,
+ sandbox: Some(sandbox.clone()),
+ });
let server = tokio::spawn(async move {
let mut websocket = accept_websocket(&listener).await;
let mut info = EnvironmentInfo::local();
@@ -2780,6 +2808,23 @@ mod tests {
Some(info),
)
.await;
+ let JSONRPCMessage::Request(write) = read_jsonrpc_websocket(&mut websocket).await
+ else {
+ panic!("expected buffered write");
+ };
+ assert_eq!(write.method, FS_WRITE_FILE_METHOD);
+ assert_eq!(
+ write.params,
+ Some(serde_json::to_value(expected_write).expect("serialize legacy write"))
+ );
+ write_jsonrpc_websocket(
+ &mut websocket,
+ JSONRPCMessage::Response(JSONRPCResponse {
+ id: write.id,
+ result: serde_json::json!({}),
+ }),
+ )
+ .await;
let JSONRPCMessage::Request(request) = read_jsonrpc_websocket(&mut websocket).await
else {
panic!("expected read-only open");
@@ -2803,12 +2848,32 @@ mod tests {
)
.await;
});
- let client = ExecServerClient::connect_websocket(RemoteExecServerConnectArgs::new(
- websocket_url,
- "file-stream-test".to_string(),
+ let remote = LazyRemoteExecServerClient::new(
+ ExecServerTransportParams::websocket_url(
+ websocket_url,
+ DEFAULT_REMOTE_EXEC_SERVER_CONNECT_TIMEOUT,
+ ),
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
- ))
- .await?;
+ );
+ let client = remote.get().await?;
+ let fs = RemoteFileSystem::new(remote);
+ // A source error precedes any legacy write, so the server must see only the complete file.
+ let broken = futures::stream::iter([
+ Ok(bytes::Bytes::from_static(b"partial")),
+ Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof)),
+ ]);
+ let error = fs
+ .write_file_stream(&read_open.path, broken.boxed(), Some(&sandbox))
+ .await
+ .expect_err("source error");
+ assert_eq!(error.kind(), std::io::ErrorKind::UnexpectedEof);
+
+ let contents = futures::stream::iter([b"\x00\xff", b"hi"])
+ .map(|chunk| Ok(bytes::Bytes::from_static(chunk)));
+ let written = fs
+ .write_file_stream(&read_open.path, contents.boxed(), Some(&sandbox))
+ .await?;
+ assert_eq!(written, 4);
let error = client
.fs_open(FsOpenParams {
mode: FsOpenMode::Replace,
@@ -2826,11 +2891,70 @@ mod tests {
.await
.unwrap_err();
assert!(matches!(error, ExecServerError::Protocol(_)));
+ // This read-only open must be the next request; there is no stream handle to close.
client.fs_open(read_open).await?;
server.await?;
Ok(())
}
+ /// An open that completes after cancellation must close that exact handle.
+ #[tokio::test]
+ async fn cancelled_streaming_replacement_closes_pending_open() -> anyhow::Result<()> {
+ let listener = TcpListener::bind("127.0.0.1:0").await?;
+ let websocket_url = format!("ws://{}", listener.local_addr()?);
+ let fs = Environment::create_for_tests(Some(websocket_url))?.get_filesystem();
+ let path = PathUri::parse("file:///tmp/cancelled.bin")?;
+ let contents = futures::stream::poll_fn(|_| panic!("cancelled write polled its source"));
+ let mut write = fs.write_file_stream(&path, contents.boxed(), /*sandbox*/ None);
+
+ let receive_open = async {
+ let mut ws = accept_websocket(&listener).await;
+ complete_websocket_initialize_with_environment_info(
+ &mut ws,
+ "session-1",
+ /*expected_resume_session_id*/ None,
+ Some(EnvironmentInfo::local()),
+ )
+ .await;
+ let JSONRPCMessage::Request(open) = read_jsonrpc_websocket(&mut ws).await else {
+ panic!("expected fs/open");
+ };
+ assert_eq!(open.method, FS_OPEN_METHOD);
+ (ws, open)
+ };
+ let (mut ws, open) = tokio::select! {
+ open = receive_open => open,
+ result = &mut write => panic!("write must wait for open: {result:?}"),
+ };
+ drop(write);
+
+ let params = open.params.expect("fs/open params");
+ let handle = params["handleId"].as_str().expect("handleId");
+ let handle = serde_json::json!({"handleId": handle});
+ write_jsonrpc_websocket(
+ &mut ws,
+ JSONRPCMessage::Response(JSONRPCResponse {
+ id: open.id,
+ result: handle.clone(),
+ }),
+ )
+ .await;
+ let JSONRPCMessage::Request(close) = read_jsonrpc_websocket(&mut ws).await else {
+ panic!("expected fs/close");
+ };
+ assert_eq!(close.method, FS_CLOSE_METHOD);
+ assert_eq!(close.params, Some(handle));
+ write_jsonrpc_websocket(
+ &mut ws,
+ JSONRPCMessage::Response(JSONRPCResponse {
+ id: close.id,
+ result: serde_json::json!({}),
+ }),
+ )
+ .await;
+ Ok(())
+ }
+
#[test_case::test_case(false; "socket_closed")]
#[test_case::test_case(true; "health_check_timed_out")]
#[tokio::test]
diff --git a/codex-rs/exec-server/src/local_file_system.rs b/codex-rs/exec-server/src/local_file_system.rs
index ec8b81c52a..abe2a0a6f1 100644
--- a/codex-rs/exec-server/src/local_file_system.rs
+++ b/codex-rs/exec-server/src/local_file_system.rs
@@ -1,3 +1,5 @@
+use bytes::Bytes;
+use codex_file_system::FILE_WRITE_CHUNK_SIZE;
use codex_file_system::MAX_WALK_DEPTH;
use codex_file_system::MAX_WALK_DIRECTORIES;
use codex_file_system::MAX_WALK_ENTRIES;
@@ -5,6 +7,8 @@ use codex_file_system::MAX_WALK_RESPONSE_BYTES;
use codex_file_system::WALK_RESPONSE_ITEM_OVERHEAD_BYTES;
use codex_utils_absolute_path::AbsolutePathBuf;
use codex_utils_path_uri::PathUri;
+use futures::StreamExt;
+use futures::stream::BoxStream;
use std::collections::HashSet;
use std::collections::VecDeque;
use std::path::Path;
@@ -14,6 +18,7 @@ use std::sync::LazyLock;
use std::time::SystemTime;
use std::time::UNIX_EPOCH;
use tokio::io;
+use tokio::io::AsyncWriteExt;
use tokio_util::io::ReaderStream;
use tokio_util::sync::CancellationToken;
@@ -253,6 +258,35 @@ impl LocalFileSystem {
}
impl ExecutorFileSystem for LocalFileSystem {
+ fn write_file_stream<'a>(
+ &'a self,
+ path: &'a PathUri,
+ mut contents: BoxStream<'a, FileSystemResult<Bytes>>,
+ sandbox: Option<&'a FileSystemSandboxContext>,
+ ) -> ExecutorFileSystemFuture<'a, u64> {
+ Box::pin(async move {
+ let mut file = self.open_file(path, FsOpenMode::Replace, sandbox).await?;
+ let mut written = 0_u64;
+ while let Some(chunk) = contents.next().await {
+ let chunk = chunk?;
+ if !(1..=FILE_WRITE_CHUNK_SIZE).contains(&chunk.len()) {
+ return Err(io::Error::new(
+ io::ErrorKind::InvalidData,
+ "invalid file stream block length",
+ ));
+ }
+ let next_offset = written.checked_add(chunk.len() as u64).ok_or_else(|| {
+ io::Error::new(io::ErrorKind::InvalidData, "file stream offset overflow")
+ })?;
+ file.write_all(&chunk).await?;
+ // Tokio can return before its blocking write finishes; finish this block before reading another.
+ file.flush().await?;
+ written = next_offset;
+ }
+ Ok(written)
+ })
+ }
+
fn canonicalize<'a>(
&'a self,
path: &'a PathUri,
diff --git a/codex-rs/exec-server/src/remote_file_system.rs b/codex-rs/exec-server/src/remote_file_system.rs
index cdcaa78bca..729b885366 100644
--- a/codex-rs/exec-server/src/remote_file_system.rs
+++ b/codex-rs/exec-server/src/remote_file_system.rs
@@ -5,13 +5,22 @@ use std::sync::MutexGuard;
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD;
+use bytes::Bytes;
+use codex_file_system::FILE_WRITE_CHUNK_SIZE;
use codex_utils_path_uri::PathUri;
+use futures::StreamExt;
+use futures::TryFutureExt;
+use futures::stream::BoxStream;
use tokio::io;
use tokio::sync::OnceCell;
+use tracing::Instrument;
+use tracing::instrument::WithSubscriber;
use tracing::trace;
+use uuid::Uuid;
use crate::CopyOptions;
use crate::CreateDirectoryOptions;
+use crate::ExecServerClient;
use crate::ExecServerError;
use crate::ExecutorFileSystem;
use crate::ExecutorFileSystemFuture;
@@ -28,13 +37,17 @@ use crate::WalkOutcome;
use crate::WriteFileOptions;
use crate::client::LazyRemoteExecServerClient;
use crate::protocol::FsCanonicalizeParams;
+use crate::protocol::FsCloseParams;
use crate::protocol::FsCopyParams;
use crate::protocol::FsCreateDirectoryParams;
use crate::protocol::FsGetMetadataParams;
+use crate::protocol::FsOpenMode;
+use crate::protocol::FsOpenParams;
use crate::protocol::FsReadDirectoryParams;
use crate::protocol::FsReadFileParams;
use crate::protocol::FsRemoveParams;
use crate::protocol::FsWalkParams;
+use crate::protocol::FsWriteBlockParams;
use crate::protocol::FsWriteFileParams;
const INVALID_REQUEST_ERROR_CODE: i64 = -32600;
@@ -317,6 +330,98 @@ impl RemoteFileSystem {
}
impl ExecutorFileSystem for RemoteFileSystem {
+ fn write_file_stream<'a>(
+ &'a self,
+ path: &'a PathUri,
+ contents: BoxStream<'a, FileSystemResult<Bytes>>,
+ sandbox: Option<&'a FileSystemSandboxContext>,
+ ) -> ExecutorFileSystemFuture<'a, u64> {
+ Box::pin(async move {
+ let client = self.client.get().await.map_err(map_remote_error)?;
+ let _invalidation = MetadataInvalidation(self);
+ let mut contents = contents.map(|chunk| {
+ let chunk = chunk?;
+ if !(1..=FILE_WRITE_CHUNK_SIZE).contains(&chunk.len()) {
+ return Err(io::Error::new(
+ io::ErrorKind::InvalidData,
+ "invalid file stream block length",
+ ));
+ }
+ Ok(chunk)
+ });
+ // Choose the legacy write before opening a stream, so an open or write failure
+ // can never cause us to replay a replacement that may have already modified the file.
+ if !client
+ .supports_file_write_streaming()
+ .await
+ .map_err(map_remote_error)?
+ {
+ let mut bytes = Vec::new();
+ while let Some(chunk) = contents.next().await {
+ bytes.extend_from_slice(&chunk?);
+ }
+ let len = bytes.len() as u64;
+ self.write_file(path, bytes, WriteFileOptions::default(), sandbox)
+ .await?;
+ return Ok(len);
+ }
+
+ // Dropping the JoinHandle lets the open finish. If it succeeds after cancellation,
+ // Tokio drops the unclaimed registration, which closes that exact handle.
+ let open = client
+ .fs_open(FsOpenParams {
+ handle_id: Uuid::new_v4().simple().to_string(),
+ path: path.clone(),
+ mode: FsOpenMode::Replace,
+ sandbox: sandbox.cloned(),
+ })
+ .map_ok(move |opened| FileWriteRegistration {
+ client,
+ handle_id: opened.handle_id,
+ active: true,
+ runtime: tokio::runtime::Handle::current(),
+ });
+ let open = tokio::spawn(open.in_current_span().with_current_subscriber());
+ let mut registration = open
+ .await
+ .map_err(io::Error::other)?
+ .map_err(map_remote_error)?;
+
+ let result: io::Result<u64> = async {
+ let mut written = 0_u64;
+ while let Some(chunk) = contents.next().await {
+ let chunk = chunk?;
+ let next_offset = written.checked_add(chunk.len() as u64).ok_or_else(|| {
+ io::Error::new(io::ErrorKind::InvalidData, "file stream offset overflow")
+ })?;
+ registration
+ .client
+ .fs_write_block(FsWriteBlockParams {
+ handle_id: registration.handle_id.clone(),
+ offset: written,
+ chunk: chunk.to_vec().into(),
+ })
+ .await
+ .map_err(map_remote_error)?;
+ written = next_offset;
+ }
+ Ok(written)
+ }
+ .await;
+ let close_result = registration
+ .client
+ .fs_close(FsCloseParams {
+ handle_id: registration.handle_id.clone(),
+ })
+ .await
+ .map_err(map_remote_error);
+ registration.active = close_result.is_err();
+ let written = result?;
+ close_result?;
+ Ok(written)
+ })
+ }
+
fn canonicalize<'a>(
&'a self,
path: &'a PathUri,
@@ -417,7 +522,36 @@ impl ExecutorFileSystem for RemoteFileSystem {
}
}
-fn map_remote_error(error: ExecServerError) -> io::Error {
+/// A cancelled write must stop later lookups from joining pre-cancellation metadata requests.
+struct MetadataInvalidation<'a>(&'a RemoteFileSystem);
+
+impl Drop for MetadataInvalidation<'_> {
+ fn drop(&mut self) {
+ self.0.lock_metadata_requests().clear();
+ }
+}
+
+/// Best-effort remote handle cleanup may outlive a cancelled transfer.
+struct FileWriteRegistration {
+ client: ExecServerClient,
+ handle_id: String,
+ active: bool,
+ runtime: tokio::runtime::Handle,
+}
+
+impl Drop for FileWriteRegistration {
+ fn drop(&mut self) {
+ if self.active {
+ let client = self.client.clone();
+ let handle_id = self.handle_id.clone();
+ self.runtime.spawn(async move {
+ let _ = client.fs_close(FsCloseParams { handle_id }).await;
+ });
+ }
+ }
+}
+
+pub(crate) fn map_remote_error(error: ExecServerError) -> io::Error {
match error {
ExecServerError::Server { code, message } if code == NOT_FOUND_ERROR_CODE => {
io::Error::new(io::ErrorKind::NotFound, message)
diff --git a/codex-rs/exec-server/tests/file_stream.rs b/codex-rs/exec-server/tests/file_stream.rs
index 2e37bb729b..8191c542cc 100644
--- a/codex-rs/exec-server/tests/file_stream.rs
+++ b/codex-rs/exec-server/tests/file_stream.rs
@@ -18,6 +18,7 @@ use codex_exec_server::RemoteExecServerConnectArgs;
use codex_http_client::HttpClientFactory;
use codex_http_client::OutboundProxyPolicy;
use codex_utils_path_uri::PathUri;
+use futures::StreamExt;
use futures::TryStreamExt;
use pretty_assertions::assert_eq;
use std::sync::Arc;
@@ -35,6 +36,140 @@ use crate::common::exec_server::exec_server;
const BLOCK_SIZE: usize = 1024 * 1024;
const OPEN_FILE_LIMIT: usize = 128;
+/// The writer must finish a block before asking its source for more data.
+#[tokio::test]
+async fn streaming_replacement_writes_before_reading_the_next_block() -> Result<()> {
+ let server = exec_server().await?;
+ let local = Environment::create_for_tests(/*exec_server_url*/ None)?.get_filesystem();
+ let remote = connect_file_system(server.websocket_url())?;
+ let tmp = TempDir::new()?;
+ let target = tmp.path().join("sequential.bin");
+ let target_uri = PathUri::from_host_native_path(&target)?;
+ for fs in [&local, &remote] {
+ // This stream borrows the target path rather than requiring an owned filesystem stream.
+ let contents = futures::stream::iter(0..3_u8).map(|block| {
+ assert_eq!(
+ std::fs::metadata(&target).unwrap().len(),
+ u64::from(block) * BLOCK_SIZE as u64
+ );
+ Ok(bytes::Bytes::from(vec![block; BLOCK_SIZE]))
+ });
+ assert_eq!(
+ fs.write_file_stream(&target_uri, contents.boxed(), /*sandbox*/ None)
+ .await?,
+ 3 * BLOCK_SIZE as u64
+ );
+ }
+ Ok(())
+}
+
+/// Streaming replacement preserves binary data across local and remote filesystem combinations.
+#[tokio::test]
+async fn streaming_replacement_copies_binary_and_empty_files() -> Result<()> {
+ let server = exec_server().await?;
+ let local = Environment::create_for_tests(/*exec_server_url*/ None)?.get_filesystem();
+ let remote = connect_file_system(server.websocket_url())?;
+ let tmp = TempDir::new()?;
+ let source = tmp.path().join("source.bin");
+ let target = tmp.path().join("target.bin");
+ let source_uri = PathUri::from_host_native_path(&source)?;
+ let target_uri = PathUri::from_host_native_path(&target)?;
+ for source_fs in [&local, &remote] {
+ for target_fs in [&local, &remote] {
+ for bytes in [[0, 255, 13, 10].repeat(BLOCK_SIZE), Vec::new()] {
+ std::fs::write(&source, &bytes)?;
+ std::fs::write(&target, vec![42; bytes.len() + 10])?;
+ let contents = source_fs
+ .read_file_stream(&source_uri, /*sandbox*/ None)
+ .await?;
+ let written = target_fs
+ .write_file_stream(&target_uri, contents.boxed(), /*sandbox*/ None)
+ .await?;
+ assert_eq!(
+ (written, std::fs::read(&target)?),
+ (bytes.len() as u64, bytes)
+ );
+ }
+ }
+ }
+ Ok(())
+}
+
+/// A failed source leaves only confirmed blocks at the target and releases remote handle capacity.
+#[tokio::test]
+async fn streaming_replacement_preserves_partial_failure_and_releases_handles() -> Result<()> {
+ let server = exec_server().await?;
+ let fs = connect_file_system(server.websocket_url())?;
+ let tmp = TempDir::new()?;
+ let target = tmp.path().join("partial.bin");
+ let target_uri = PathUri::from_host_native_path(&target)?;
+ for _ in 0..=OPEN_FILE_LIMIT {
+ std::fs::write(&target, b"original target contents")?;
+ let contents = futures::stream::iter([
+ Ok(bytes::Bytes::from_static(b"partial")),
+ Err(std::io::Error::other("source failed")),
+ ]);
+ let error = fs
+ .write_file_stream(&target_uri, contents.boxed(), /*sandbox*/ None)
+ .await
+ .unwrap_err();
+ assert_eq!(
+ (error.to_string(), std::fs::read(&target)?),
+ ("source failed".to_string(), b"partial".to_vec())
+ );
+ }
+ Ok(())
+}
+
+/// Cancelling a partial replacement must not let later metadata lookups reuse an older response.
+#[tokio::test]
+async fn cancelled_streaming_replacement_invalidates_metadata() -> Result<()> {
+ let server = exec_server().await?;
+ let fs = connect_file_system(server.websocket_url())?;
+ let tmp = TempDir::new()?;
+ let target = tmp.path().join("metadata.bin");
+ let path = PathUri::from_host_native_path(&target)?;
+ let original = b"original target contents";
+ std::fs::write(&target, original)?;
+ // Connect before polling metadata so its first poll sends the request.
+ fs.read_file(&path, Default::default(), /*sandbox*/ None)
+ .await?;
+ let stale = fs.get_metadata(&path, Default::default(), /*sandbox*/ None);
+ tokio::pin!(stale);
+ assert!(futures::poll!(stale.as_mut()).is_pending());
+ // The fixture dispatches requests in order. This read confirms the old stat finished,
+ // while leaving its caller unpolled so that another lookup could still join it.
+ assert_eq!(
+ fs.read_file(&path, Default::default(), /*sandbox*/ None)
+ .await?,
+ original
+ );
+
+ let (polled_tx, polled_rx) = tokio::sync::oneshot::channel();
+ let contents = futures::stream::iter([Ok(bytes::Bytes::from_static(b"partial"))]).chain(
+ futures::stream::once(async move {
+ let _ = polled_tx.send(());
+ futures::future::pending::<std::io::Result<bytes::Bytes>>().await
+ }),
+ );
+ let mut write = fs.write_file_stream(&path, contents.boxed(), /*sandbox*/ None);
+ tokio::select! {
+ result = &mut write => panic!("copy should await its source: {result:?}"),
+ _ = polled_rx => {},
+ }
+ drop(write);
+
+ let (refreshed, stale) = tokio::join!(
+ fs.get_metadata(&path, Default::default(), /*sandbox*/ None),
+ stale.as_mut(),
+ );
+ assert_eq!(
+ (stale?.size, refreshed?.size, std::fs::read(&target)?),
+ (original.len() as u64, 7, b"partial".to_vec())
+ );
+ Ok(())
+}
+
#[tokio::test]
async fn stream_stops_after_an_exact_block_boundary() -> Result<()> {
let server = exec_server().await?;
diff --git a/codex-rs/file-system/src/environment_accessor.rs b/codex-rs/file-system/src/environment_accessor.rs
index 159edeb446..06f63ede9e 100644
--- a/codex-rs/file-system/src/environment_accessor.rs
+++ b/codex-rs/file-system/src/environment_accessor.rs
@@ -6,6 +6,7 @@ use crate::ExecutorFileSystem;
use crate::ExecutorFileSystemFuture;
use crate::FileMetadata;
use crate::FileSystemReadStream;
+use crate::FileSystemResult;
use crate::FileSystemSandboxContext;
use crate::GetMetadataOptions;
use crate::ReadDirectoryEntry;
@@ -14,7 +15,9 @@ use crate::RemoveOptions;
use crate::WalkOptions;
use crate::WalkOutcome;
use crate::WriteFileOptions;
+use bytes::Bytes;
use codex_utils_path_uri::PathUri;
+use futures::stream::BoxStream;
use std::fmt;
use std::hash::Hash;
use std::hash::Hasher;
@@ -43,6 +46,13 @@ pub trait EnvironmentAccess: Send + Sync {
path: &'a PathUri,
) -> ExecutorFileSystemFuture<'a, FileSystemReadStream>;
+ /// Replaces a file through [`ExecutorFileSystem::write_file_stream`] with captured permissions.
+ fn write_file_stream<'a>(
+ &'a self,
+ path: &'a PathUri,
+ contents: BoxStream<'a, FileSystemResult<Bytes>>,
+ ) -> ExecutorFileSystemFuture<'a, u64>;
+
fn write_file<'a>(
&'a self,
path: &'a PathUri,
@@ -210,6 +220,15 @@ impl EnvironmentAccess for FileSystemEnvironmentAccessor<'_> {
.read_file_stream(path, self.key.sandbox.as_deref())
}
+ fn write_file_stream<'a>(
+ &'a self,
+ path: &'a PathUri,
+ contents: BoxStream<'a, FileSystemResult<Bytes>>,
+ ) -> ExecutorFileSystemFuture<'a, u64> {
+ self.file_system
+ .write_file_stream(path, contents, self.key.sandbox.as_deref())
+ }
+
fn write_file<'a>(
&'a self,
path: &'a PathUri,
diff --git a/codex-rs/file-system/src/lib.rs b/codex-rs/file-system/src/lib.rs
index 1687d006f5..7b95299b5e 100644
--- a/codex-rs/file-system/src/lib.rs
+++ b/codex-rs/file-system/src/lib.rs
@@ -29,6 +29,7 @@ pub use find_up::FindUpErrorPolicy;
pub use find_up::find_nearest_ancestor_with_markers;
pub use find_up::find_nearest_native_ancestor_with_markers;
use futures::Stream;
+use futures::stream::BoxStream;
pub use nfs::is_nfs;
use serde::Deserialize;
use serde::Serialize;
@@ -660,6 +661,36 @@ pub trait ExecutorFileSystem: Send + Sync {
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, FileSystemReadStream>;
+ /// Replaces a regular file from a stream of bounded chunks.
+ ///
+ /// Accepts any sendable byte stream, boxed to keep this trait object-safe.
+ /// Input chunks must be nonempty and no larger than [`FILE_WRITE_CHUNK_SIZE`].
+ /// Returns the number of bytes written after closing the target. Parents must already exist.
+ /// Failure or cancellation may leave a truncated or partial target.
+ ///
+ /// Cancellation is best effort: dropping the future stops consuming input and issuing new
+ /// blocks, but an in-flight open or write may still truncate or modify the target. Dropping
+ /// does not wait for that I/O or remote handle cleanup, so a subsequent replacement may overlap
+ /// with the abandoned operation.
+ ///
+ /// Local and streaming-capable remote filesystems write one chunk at a time. For compatibility,
+ /// a remote filesystem may buffer the entire input and send one write to an older executor.
+ /// The default implementation returns `Unsupported` without consuming the input.
+ fn write_file_stream<'a>(
+ &'a self,
+ path: &'a PathUri,
+ contents: BoxStream<'a, FileSystemResult<Bytes>>,
+ sandbox: Option<&'a FileSystemSandboxContext>,
+ ) -> ExecutorFileSystemFuture<'a, u64> {
+ Box::pin(async move {
+ let _ = (path, contents, sandbox);
+ Err(io::Error::new(
+ io::ErrorKind::Unsupported,
+ "filesystem does not support streaming writes",
+ ))
+ })
+ }
+
/// Reads a file and decodes it as UTF-8 text.
fn read_file_text<'a>(
&'a self,