Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(athena): type Hive partitions from Glue and resolve db.table.column references (#2113)
Mayuri99bba52ddac7
diff --git a/ministack/services/athena.py b/ministack/services/athena.pyindex d1182edb..1718440c 100644--- a/ministack/services/athena.py+++ b/ministack/services/athena.py@@ -978,12 +978,15 @@ async def _rewrite_data_paths(query, database):stripped = f"{p.netloc}{p.path}".rstrip("/")local_dir = f"{ATHENA_DATA_DIR}/{account_id}/{stripped}"if next(glob.iglob(f"{local_dir}/**/*", recursive=True), None):- relation = f"'{local_dir}/**/*.{_data_format(table_data)}'" # DuckDB reads the files, or reports why not+ relation = _file_relation(f"{local_dir}/**/*.{_data_format(table_data)}", table_data)else:relation = _empty_relation(table_data)alias = "" if ref.aliased else f' AS "{ref.table}"'edits.append((ref.start, ref.end, relation + alias))+ if not ref.aliased:+ edits.extend(_qualified_column_prefixes(query, db_name, ref.table, edits[-1]))+ edits = sorted(set(edits)) # a table named twice yields its column-prefix edits twicefor span_start, span_end, replacement in reversed(edits):query = query[:span_start] + replacement + query[span_end:]return _rewrite_s3_paths(query)@@ -996,6 +999,56 @@ _DUCKDB_TYPE_BY_ATHENA = {}+# Readers that accept ``hive_types``, so partition columns get their Glue types.+_HIVE_TYPED_READERS = {"parquet": "read_parquet", "csv": "read_csv", "json": "read_json"}+++def _file_relation(pattern, table_data):+ """The table's files; partition keys are typed from Glue instead of inferred from the path."""+ path = "'" + pattern.replace("'", "''") + "'"+ reader = _HIVE_TYPED_READERS.get(_data_format(table_data))+ partition_keys = table_data.get("PartitionKeys") or []+ if not reader or not partition_keys:+ return path # DuckDB reads the files, or reports why not+ hive_types = ", ".join(+ "'{}': {}".format(str(key.get("Name", "")).replace("'", "''"), _duckdb_type(key)) for key in partition_keys+ )+ return f"{reader}({path}, hive_partitioning = true, hive_types = {{{hive_types}}})"+++def _duckdb_type(column):+ base = str(column.get("Type", "string")).lower().split("(")[0].strip()+ return "DECIMAL" if base == "decimal" else _DUCKDB_TYPE_BY_ATHENA.get(base, "VARCHAR")+++def _qualified_column_prefixes(query, database, table, table_edit):+ """Edits turning ``database.table.column`` (optionally catalog-qualified) into ``"table".column``.++ The table itself is replaced by a relation aliased ``"table"``, so DuckDB cannot+ resolve column references qualified with the database name.+ """+ tokens = list(_sql_tokens(query))+ edits = []+ for i in range(len(tokens) - 4):+ if table_edit[0] <= tokens[i][2] < table_edit[1]:+ continue # the FROM/JOIN reference itself+ if i and tokens[i - 1][:2] == ("punct", "."):+ continue+ names, j = [], i+ while j < len(tokens) and tokens[j][0] in ("name", "qname"):+ names.append(tokens[j])+ if j + 1 < len(tokens) and tokens[j + 1][:2] == ("punct", "."):+ j += 2+ else:+ break+ if len(names) not in (3, 4):+ continue+ db_token, table_token = names[-3], names[-2]+ if db_token[1] == database and table_token[1] == table:+ edits.append((names[0][2], table_token[3], '"{}"'.format(table.replace('"', '""'))))+ return edits++def _empty_relation(table_data):"""Zero rows with the table's Glue columns and partition keys; complex types read as VARCHAR."""columns = ((table_data.get("StorageDescriptor") or {}).get("Columns") or []) + (table_data.get("PartitionKeys") or [])@@ -1003,10 +1056,8 @@ def _empty_relation(table_data):return "(SELECT 1 WHERE FALSE)"projected = []for column in columns:- base = str(column.get("Type", "string")).lower().split("(")[0].strip()- duck_type = "DECIMAL" if base == "decimal" else _DUCKDB_TYPE_BY_ATHENA.get(base, "VARCHAR")name = str(column.get("Name", "")).replace('"', '""')- projected.append(f'CAST(NULL AS {duck_type}) AS "{name}"')+ projected.append(f'CAST(NULL AS {_duckdb_type(column)}) AS "{name}"')return f"(SELECT {', '.join(projected)} WHERE FALSE)"diff --git a/tests/test_athena.py b/tests/test_athena.pyindex 38d6790d..098ebda4 100644--- a/tests/test_athena.py+++ b/tests/test_athena.py@@ -1082,6 +1082,43 @@ def test_athena_reads_a_parquet_table_created_without_classification(assert [row["Data"][0]["VarCharValue"] for row in rows] == ["id", "1", "2"]+def test_athena_partitioned_table_types_partitions_and_resolves_qualified_columns(+ athena, glue, s3, persisted_parquet_store, tmp_path,+):+ """Partition keys take their Glue types, and ``db.table.column`` resolves against the table."""+ suffix = _uuid_mod.uuid4().hex[:10]+ bucket, database = f"athena-partitions-{suffix}", f"partitions_{suffix}"+ s3.create_bucket(Bucket=bucket)+ glue.create_database(DatabaseInput={"Name": database})+ parquet = tmp_path / "event.parquet"+ with duckdb.connect() as connection:+ connection.execute("COPY (SELECT 'a' AS note, 1 AS action) TO '%s' (FORMAT PARQUET)" % parquet)+ persisted_parquet_store._persist_object(+ bucket, "usage/year=2026/month=04/day=23/event.parquet", parquet.read_bytes(),+ )+ glue.create_table(DatabaseName=database, TableInput={+ "Name": "usage",+ "StorageDescriptor": {+ "Location": f"s3://{bucket}/usage/",+ "Columns": [{"Name": "note", "Type": "string"}, {"Name": "action", "Type": "int"}],+ },+ "PartitionKeys": [+ {"Name": "year", "Type": "string"},+ {"Name": "month", "Type": "string"},+ {"Name": "day", "Type": "string"},+ ],+ "Parameters": {"classification": "parquet"},+ })++ query_id, execution = _run_to_completion(+ athena, f"SELECT {database}.usage.note, year, month, day FROM {database}.usage", database,+ )+ assert execution["Status"]["State"] == "SUCCEEDED", execution["Status"]+ result = athena.get_query_results(QueryExecutionId=query_id)["ResultSet"]+ assert [cell["VarCharValue"] for cell in result["Rows"][1]["Data"]] == ["a", "2026", "04", "23"]+ assert [column["Type"] for column in result["ResultSetMetadata"]["ColumnInfo"]] == ["varchar"] * 4++# ---- DDL and table-reference parsing (in-process, no server) ----