Apache DataFusion SQL Query Engine
#!/usr/bin/env bash
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
# This script is meant for developers of DataFusion -- it is runnable
# from the standard DataFusion development environment and uses cargo,
# etc and orchestrates gathering data and run the benchmark binary in
# different configurations.
# Script to download ClickBench queries and split them into individual files
set -e # Exit on any error
# URL for the raw file (not the GitHub page)
URL="https://raw.githubusercontent.com/ClickHouse/ClickBench/main/datafusion/queries.sql"
# Temporary file to store downloaded content
TEMP_FILE="queries.sql"
TARGET_DIR="queries"
# Download the file
echo "Downloading queries from $URL..."
if command -v curl &> /dev/null; then
curl -s -o "$TEMP_FILE" "$URL"
elif command -v wget &> /dev/null; then
wget -q -O "$TEMP_FILE" "$URL"
else
echo "Error: Neither curl nor wget is available. Please install one of them."
exit 1
fi
# Check if download was successful
if [ ! -f "$TEMP_FILE" ] || [ ! -s "$TEMP_FILE" ]; then
echo "Error: Failed to download or file is empty"
exit 1
fi
# Initialize counter
counter=0
# Ensure the target directory exists
if [ ! -d ${TARGET_DIR} ]; then
mkdir -p ${TARGET_DIR}
fi
# Read the file line by line and create individual query files
mapfile -t lines < $TEMP_FILE
for line in "${lines[@]}"; do
# Skip empty lines
if [ -n "$line" ]; then
# Create filename with zero-padded counter
filename="q${counter}.sql"
# Write the line to the individual file
echo "$line" > "${TARGET_DIR}/$filename"
echo "Created ${TARGET_DIR}/$filename"
# Increment counter
(( counter += 1 ))
fi
done
# Clean up temporary file
rm "$TEMP_FILE"