Apache DataFusion SQL Query Engine
Adds instrumentation to delimited LIST operations in CLI (#18134)
Blake Orthec3d20bff605
diff --git a/datafusion-cli/src/object_storage/instrumented.rs b/datafusion-cli/src/object_storage/instrumented.rsindex 8acece315f..94445ee64e 100644--- a/datafusion-cli/src/object_storage/instrumented.rs+++ b/datafusion-cli/src/object_storage/instrumented.rs@@ -163,6 +163,28 @@ impl InstrumentedObjectStore {ret}++ async fn instrumented_list_with_delimiter(+ &self,+ prefix: Option<&Path>,+ ) -> Result<ListResult> {+ let timestamp = Utc::now();+ let start = Instant::now();+ let ret = self.inner.list_with_delimiter(prefix).await?;+ let elapsed = start.elapsed();++ self.requests.lock().push(RequestDetails {+ op: Operation::List,+ path: prefix.cloned().unwrap_or_else(|| Path::from("")),+ timestamp,+ duration: Some(elapsed),+ size: None,+ range: None,+ extra_display: None,+ });++ Ok(ret)+ }}impl fmt::Display for InstrumentedObjectStore {@@ -217,6 +239,10 @@ impl ObjectStore for InstrumentedObjectStore {}async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {+ if self.enabled() {+ return self.instrumented_list_with_delimiter(prefix).await;+ }+self.inner.list_with_delimiter(prefix).await}@@ -569,6 +595,29 @@ mod tests {assert!(request.extra_display.is_none());}+ #[tokio::test]+ async fn instrumented_store_list_with_delimiter() {+ let (instrumented, path) = setup_test_store().await;++ // By default no requests should be instrumented/stored+ assert!(instrumented.requests.lock().is_empty());+ let _ = instrumented.list_with_delimiter(Some(&path)).await.unwrap();+ assert!(instrumented.requests.lock().is_empty());++ instrumented.set_instrument_mode(InstrumentedObjectStoreMode::Trace);+ assert!(instrumented.requests.lock().is_empty());+ let _ = instrumented.list_with_delimiter(Some(&path)).await.unwrap();+ assert_eq!(instrumented.requests.lock().len(), 1);++ let request = instrumented.take_requests().pop().unwrap();+ assert_eq!(request.op, Operation::List);+ assert_eq!(request.path, path);+ assert!(request.duration.is_some());+ assert!(request.size.is_none());+ assert!(request.range.is_none());+ assert!(request.extra_display.is_none());+ }+#[test]fn request_details() {let rd = RequestDetails {