Skip to content

Apache DataFusion SQL Query Engine

2.1 KB
Raw Blame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

//! See `main.rs` for how to run it.

use datafusion::error::Result;
use datafusion::prelude::*;
use object_store::aws::AmazonS3Builder;
use std::env;
use std::sync::Arc;
use url::Url;

/// This example demonstrates querying data in a public S3 bucket
/// (the NYC TLC open dataset: `s3://nyc-tlc`).
///
/// The following environment variables must be defined:
/// - `AWS_ACCESS_KEY_ID`
/// - `AWS_SECRET_ACCESS_KEY`
pub async fn query_aws_s3() -> Result<()> {
    let ctx = SessionContext::new();

    // the region must be set to the region where the bucket exists until the following
    // issue is resolved
    // https://github.com/apache/arrow-rs/issues/2795
    let region = "us-east-1";
    let bucket_name = "nyc-tlc";

    let s3 = AmazonS3Builder::new()
        .with_bucket_name(bucket_name)
        .with_region(region)
        .with_access_key_id(env::var("AWS_ACCESS_KEY_ID").unwrap())
        .with_secret_access_key(env::var("AWS_SECRET_ACCESS_KEY").unwrap())
        .build()?;

    let path = format!("s3://{bucket_name}");
    let s3_url = Url::parse(&path).unwrap();
    ctx.register_object_store(&s3_url, Arc::new(s3));

    // cannot query the parquet files from this bucket because the path contains a whitespace
    // and we don't support that yet
    // https://github.com/apache/arrow-rs/issues/2799
    let path = format!("s3://{bucket_name}/csv_backup/yellow_tripdata_2022-02.csv");
    ctx.register_csv("trips", &path, CsvReadOptions::default())
        .await?;

    // execute the query
    let df = ctx.sql("SELECT * FROM trips LIMIT 10").await?;

    // print the results
    df.show().await?;

    // dynamic query by the file path
    let ctx = ctx.enable_url_table();
    let df = ctx
        .sql(format!(r#"SELECT * FROM '{path}' LIMIT 10"#).as_str())
        .await?;

    // print the results
    df.show().await?;

    Ok(())
}