Skip to content

Apache DataFusion SQL Query Engine

[datafusion-cli] Implement average LIST duration for object store profiling (#19127)

Peter Lee2c9077a8bfd0
diff --git a/datafusion-cli/src/object_storage/instrumented.rs b/datafusion-cli/src/object_storage/instrumented.rs
index a6d05a8dcc..0d5e9dc2c5 100644
--- a/datafusion-cli/src/object_storage/instrumented.rs
+++ b/datafusion-cli/src/object_storage/instrumented.rs
@@ -35,7 +35,7 @@ use datafusion::{
error::DataFusionError,
execution::object_store::{DefaultObjectStoreRegistry, ObjectStoreRegistry},
};
-use futures::stream::BoxStream;
+use futures::stream::{BoxStream, Stream};
use object_store::{
GetOptions, GetRange, GetResult, ListResult, MultipartUpload, ObjectMeta,
ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result,
@@ -44,6 +44,58 @@ use object_store::{
use parking_lot::{Mutex, RwLock};
use url::Url;
+/// A stream wrapper that measures the time until the first response(item or end of stream) is yielded
+struct TimeToFirstItemStream<S> {
+ inner: S,
+ start: Instant,
+ request_index: usize,
+ requests: Arc<Mutex<Vec<RequestDetails>>>,
+ first_item_yielded: bool,
+}
+
+impl<S> TimeToFirstItemStream<S> {
+ fn new(
+ inner: S,
+ start: Instant,
+ request_index: usize,
+ requests: Arc<Mutex<Vec<RequestDetails>>>,
+ ) -> Self {
+ Self {
+ inner,
+ start,
+ request_index,
+ requests,
+ first_item_yielded: false,
+ }
+ }
+}
+
+impl<S> Stream for TimeToFirstItemStream<S>
+where
+ S: Stream<Item = Result<ObjectMeta>> + Unpin,
+{
+ type Item = Result<ObjectMeta>;
+
+ fn poll_next(
+ mut self: std::pin::Pin<&mut Self>,
+ cx: &mut std::task::Context<'_>,
+ ) -> std::task::Poll<Option<Self::Item>> {
+ let poll_result = std::pin::Pin::new(&mut self.inner).poll_next(cx);
+
+ if !self.first_item_yielded && poll_result.is_ready() {
+ self.first_item_yielded = true;
+ let elapsed = self.start.elapsed();
+
+ let mut requests = self.requests.lock();
+ if let Some(request) = requests.get_mut(self.request_index) {
+ request.duration = Some(elapsed);
+ }
+ }
+
+ poll_result
+ }
+}
+
/// The profiling mode to use for an [`InstrumentedObjectStore`] instance. Collecting profiling
/// data will have a small negative impact on both CPU and memory usage. Default is `Disabled`
#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)]
@@ -92,7 +144,7 @@ impl From<u8> for InstrumentedObjectStoreMode {
pub struct InstrumentedObjectStore {
inner: Arc<dyn ObjectStore>,
instrument_mode: AtomicU8,
- requests: Mutex<Vec<RequestDetails>>,
+ requests: Arc<Mutex<Vec<RequestDetails>>>,
}
impl InstrumentedObjectStore {
@@ -101,7 +153,7 @@ impl InstrumentedObjectStore {
Self {
inner: object_store,
instrument_mode,
- requests: Mutex::new(Vec::new()),
+ requests: Arc::new(Mutex::new(Vec::new())),
}
}
@@ -219,19 +271,31 @@ impl InstrumentedObjectStore {
prefix: Option<&Path>,
) -> BoxStream<'static, Result<ObjectMeta>> {
let timestamp = Utc::now();
- let ret = self.inner.list(prefix);
+ let start = Instant::now();
+ let inner_stream = self.inner.list(prefix);
+
+ let request_index = {
+ let mut requests = self.requests.lock();
+ requests.push(RequestDetails {
+ op: Operation::List,
+ path: prefix.cloned().unwrap_or_else(|| Path::from("")),
+ timestamp,
+ duration: None,
+ size: None,
+ range: None,
+ extra_display: None,
+ });
+ requests.len() - 1
+ };
- self.requests.lock().push(RequestDetails {
- op: Operation::List,
- path: prefix.cloned().unwrap_or_else(|| Path::from("")),
- timestamp,
- duration: None, // list returns a stream, so the duration isn't meaningful
- size: None,
- range: None,
- extra_display: None,
- });
+ let wrapped_stream = TimeToFirstItemStream::new(
+ inner_stream,
+ start,
+ request_index,
+ Arc::clone(&self.requests),
+ );
- ret
+ Box::pin(wrapped_stream)
}
async fn instrumented_list_with_delimiter(
@@ -759,6 +823,7 @@ impl ObjectStoreRegistry for InstrumentedObjectStoreRegistry {
#[cfg(test)]
mod tests {
+ use futures::StreamExt;
use object_store::WriteMultipart;
use super::*;
@@ -899,13 +964,15 @@ mod tests {
instrumented.set_instrument_mode(InstrumentedObjectStoreMode::Trace);
assert!(instrumented.requests.lock().is_empty());
- let _ = instrumented.list(Some(&path));
+ let mut stream = instrumented.list(Some(&path));
+ // Consume at least one item from the stream to trigger duration measurement
+ let _ = stream.next().await;
assert_eq!(instrumented.requests.lock().len(), 1);
let request = instrumented.take_requests().pop().unwrap();
assert_eq!(request.op, Operation::List);
assert_eq!(request.path, path);
- assert!(request.duration.is_none());
+ assert!(request.duration.is_some());
assert!(request.size.is_none());
assert!(request.range.is_none());
assert!(request.extra_display.is_none());