Skip to content

Apache DataFusion SQL Query Engine

2.0 KB
Raw Blame
#!/usr/bin/env bash
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

# This script is meant for developers of DataFusion -- it is runnable
# from the standard DataFusion development environment and uses cargo,
# etc and orchestrates gathering data and run the benchmark binary in
# different configurations.

# Script to download ClickBench queries and split them into individual files

set -e  # Exit on any error

# URL for the raw file (not the GitHub page)
URL="https://raw.githubusercontent.com/ClickHouse/ClickBench/main/datafusion/queries.sql"

# Temporary file to store downloaded content
TEMP_FILE="queries.sql"

TARGET_DIR="queries"

# Download the file
echo "Downloading queries from $URL..."
if command -v curl &> /dev/null; then
    curl -s -o "$TEMP_FILE" "$URL"
elif command -v wget &> /dev/null; then
    wget -q -O "$TEMP_FILE" "$URL"
else
    echo "Error: Neither curl nor wget is available. Please install one of them."
    exit 1
fi

# Check if download was successful
if [ ! -f "$TEMP_FILE" ] || [ ! -s "$TEMP_FILE" ]; then
    echo "Error: Failed to download or file is empty"
    exit 1
fi

# Initialize counter
counter=0

# Ensure the target directory exists
if [ ! -d ${TARGET_DIR} ]; then
  mkdir -p ${TARGET_DIR}
fi

# Read the file line by line and create individual query files
mapfile -t lines < $TEMP_FILE
for line in "${lines[@]}"; do
    # Skip empty lines
    if [ -n "$line" ]; then
        # Create filename with zero-padded counter
        filename="q${counter}.sql"

        # Write the line to the individual file
        echo "$line" > "${TARGET_DIR}/$filename"

        echo "Created ${TARGET_DIR}/$filename"

        # Increment counter
        (( counter += 1 ))
    fi
done

# Clean up temporary file
rm "$TEMP_FILE"