Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

test(athena): cover Glue-backed Parquet query (#1848)

Lisandro Pintos244897304cd2
diff --git a/tests/conftest.py b/tests/conftest.py
index 2c1766b8..7cf798a6 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -99,6 +99,7 @@ def sqs_policy_allow_sns(queue_arn, topic_arn):
_SERIAL_TESTS = {
+ "tests/test_athena.py::test_athena_queries_glue_backed_parquet",
"tests/test_rds.py::test_rds_pg_two_replicating_readers_provision_source_once",
"tests/test_athena.py::test_athena_engine_mock_via_config",
"tests/test_athena.py::test_athena_mixed_glue_and_s3_uri",
diff --git a/tests/test_athena.py b/tests/test_athena.py
index 4f757ed3..fd88c6d1 100644
--- a/tests/test_athena.py
+++ b/tests/test_athena.py
@@ -1,14 +1,88 @@
import json
import os
import time
+import urllib.request
import uuid as _uuid_mod
import boto3
+import duckdb
import pytest
from botocore.config import Config
from botocore.exceptions import ClientError
+def persisted_parquet_store(monkeypatch, tmp_path):
+ """Make a local Parquet file visible to MiniStack's Glue-backed Athena reader."""
+ from ministack.services import s3 as s3mod
+
+ monkeypatch.setattr(s3mod, "DATA_DIR", str(tmp_path))
+ monkeypatch.setattr(s3mod, "S3_PERSIST", True)
+ endpoint = os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566")
+
+ def configure(path):
+ request = urllib.request.Request(
+ f"{endpoint}/_ministack/config",
+ data=json.dumps({"athena.ATHENA_DATA_DIR": path}).encode(),
+ headers={"Content-Type": "application/json"},
+ method="POST",
+ )
+ urllib.request.urlopen(request, timeout=5).close()
+
+ configure(str(tmp_path))
+ try:
+ yield s3mod
+ finally:
+ configure(os.environ.get("S3_DATA_DIR", "/tmp/ministack-data/s3"))
+
+
+def test_athena_queries_glue_backed_parquet(
+ athena, glue, s3, persisted_parquet_store, tmp_path,
+):
+ """Resolve a Glue Parquet table and apply a predicate to its rows."""
+ suffix = _uuid_mod.uuid4().hex[:10]
+ bucket = f"athena-parquet-{suffix}"
+ database = f"parquet_{suffix}"
+ s3.create_bucket(Bucket=bucket)
+ glue.create_database(DatabaseInput={"Name": database})
+
+ parquet = tmp_path / "items.parquet"
+ with duckdb.connect() as connection:
+ connection.execute("CREATE TABLE items (id INTEGER, category VARCHAR)")
+ connection.executemany(
+ "INSERT INTO items VALUES (?, ?)", [(1, "keep"), (2, "skip"), (3, "keep")],
+ )
+ connection.execute(f"COPY items TO '{parquet}' (FORMAT PARQUET)")
+ persisted_parquet_store._persist_object(
+ bucket, "data/items.parquet", parquet.read_bytes(),
+ )
+ glue.create_table(DatabaseName=database, TableInput={
+ "Name": "items",
+ "StorageDescriptor": {
+ "Location": f"s3://{bucket}/data/",
+ "Columns": [
+ {"Name": "id", "Type": "int"},
+ {"Name": "category", "Type": "string"},
+ ],
+ },
+ "Parameters": {"classification": "parquet"},
+ })
+ query_id = athena.start_query_execution(
+ QueryString=(f"SELECT id FROM {database}.items "
+ "WHERE category = 'keep' ORDER BY id"),
+ QueryExecutionContext={"Database": database},
+ ResultConfiguration={"OutputLocation": f"s3://{bucket}/results/"},
+ )["QueryExecutionId"]
+ for _ in range(30):
+ execution = athena.get_query_execution(QueryExecutionId=query_id)["QueryExecution"]
+ if execution["Status"]["State"] in {"SUCCEEDED", "FAILED", "CANCELLED"}:
+ break
+ time.sleep(0.1)
+ assert execution["Status"]["State"] == "SUCCEEDED", execution["Status"]
+ rows = athena.get_query_results(QueryExecutionId=query_id)["ResultSet"]["Rows"]
+ assert [row["Data"][0]["VarCharValue"] for row in rows] == ["id", "1", "3"]
+
+
def _client(region):
return boto3.client(
"athena",