Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
test(athena): cover Glue-backed Parquet query (#1848)
Lisandro Pintos244897304cd2
diff --git a/tests/conftest.py b/tests/conftest.pyindex 2c1766b8..7cf798a6 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -99,6 +99,7 @@ def sqs_policy_allow_sns(queue_arn, topic_arn):_SERIAL_TESTS = {+ "tests/test_athena.py::test_athena_queries_glue_backed_parquet","tests/test_rds.py::test_rds_pg_two_replicating_readers_provision_source_once","tests/test_athena.py::test_athena_engine_mock_via_config","tests/test_athena.py::test_athena_mixed_glue_and_s3_uri",diff --git a/tests/test_athena.py b/tests/test_athena.pyindex 4f757ed3..fd88c6d1 100644--- a/tests/test_athena.py+++ b/tests/test_athena.py@@ -1,14 +1,88 @@import jsonimport osimport time+import urllib.requestimport uuid as _uuid_modimport boto3+import duckdbimport pytestfrom botocore.config import Configfrom botocore.exceptions import ClientError+def persisted_parquet_store(monkeypatch, tmp_path):+ """Make a local Parquet file visible to MiniStack's Glue-backed Athena reader."""+ from ministack.services import s3 as s3mod++ monkeypatch.setattr(s3mod, "DATA_DIR", str(tmp_path))+ monkeypatch.setattr(s3mod, "S3_PERSIST", True)+ endpoint = os.environ.get("MINISTACK_ENDPOINT", "http://localhost:4566")++ def configure(path):+ request = urllib.request.Request(+ f"{endpoint}/_ministack/config",+ data=json.dumps({"athena.ATHENA_DATA_DIR": path}).encode(),+ headers={"Content-Type": "application/json"},+ method="POST",+ )+ urllib.request.urlopen(request, timeout=5).close()++ configure(str(tmp_path))+ try:+ yield s3mod+ finally:+ configure(os.environ.get("S3_DATA_DIR", "/tmp/ministack-data/s3"))+++def test_athena_queries_glue_backed_parquet(+ athena, glue, s3, persisted_parquet_store, tmp_path,+):+ """Resolve a Glue Parquet table and apply a predicate to its rows."""+ suffix = _uuid_mod.uuid4().hex[:10]+ bucket = f"athena-parquet-{suffix}"+ database = f"parquet_{suffix}"+ s3.create_bucket(Bucket=bucket)+ glue.create_database(DatabaseInput={"Name": database})++ parquet = tmp_path / "items.parquet"+ with duckdb.connect() as connection:+ connection.execute("CREATE TABLE items (id INTEGER, category VARCHAR)")+ connection.executemany(+ "INSERT INTO items VALUES (?, ?)", [(1, "keep"), (2, "skip"), (3, "keep")],+ )+ connection.execute(f"COPY items TO '{parquet}' (FORMAT PARQUET)")+ persisted_parquet_store._persist_object(+ bucket, "data/items.parquet", parquet.read_bytes(),+ )+ glue.create_table(DatabaseName=database, TableInput={+ "Name": "items",+ "StorageDescriptor": {+ "Location": f"s3://{bucket}/data/",+ "Columns": [+ {"Name": "id", "Type": "int"},+ {"Name": "category", "Type": "string"},+ ],+ },+ "Parameters": {"classification": "parquet"},+ })+ query_id = athena.start_query_execution(+ QueryString=(f"SELECT id FROM {database}.items "+ "WHERE category = 'keep' ORDER BY id"),+ QueryExecutionContext={"Database": database},+ ResultConfiguration={"OutputLocation": f"s3://{bucket}/results/"},+ )["QueryExecutionId"]+ for _ in range(30):+ execution = athena.get_query_execution(QueryExecutionId=query_id)["QueryExecution"]+ if execution["Status"]["State"] in {"SUCCEEDED", "FAILED", "CANCELLED"}:+ break+ time.sleep(0.1)+ assert execution["Status"]["State"] == "SUCCEEDED", execution["Status"]+ rows = athena.get_query_results(QueryExecutionId=query_id)["ResultSet"]["Rows"]+ assert [row["Data"][0]["VarCharValue"] for row in rows] == ["id", "1", "3"]++def _client(region):return boto3.client("athena",