Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

fix(athena): type Hive partitions from Glue and resolve db.table.column references (#2113)

Mayuri99bba52ddac7
diff --git a/ministack/services/athena.py b/ministack/services/athena.py
index d1182edb..1718440c 100644
--- a/ministack/services/athena.py
+++ b/ministack/services/athena.py
@@ -978,12 +978,15 @@ async def _rewrite_data_paths(query, database):
stripped = f"{p.netloc}{p.path}".rstrip("/")
local_dir = f"{ATHENA_DATA_DIR}/{account_id}/{stripped}"
if next(glob.iglob(f"{local_dir}/**/*", recursive=True), None):
- relation = f"'{local_dir}/**/*.{_data_format(table_data)}'" # DuckDB reads the files, or reports why not
+ relation = _file_relation(f"{local_dir}/**/*.{_data_format(table_data)}", table_data)
else:
relation = _empty_relation(table_data)
alias = "" if ref.aliased else f' AS "{ref.table}"'
edits.append((ref.start, ref.end, relation + alias))
+ if not ref.aliased:
+ edits.extend(_qualified_column_prefixes(query, db_name, ref.table, edits[-1]))
+ edits = sorted(set(edits)) # a table named twice yields its column-prefix edits twice
for span_start, span_end, replacement in reversed(edits):
query = query[:span_start] + replacement + query[span_end:]
return _rewrite_s3_paths(query)
@@ -996,6 +999,56 @@ _DUCKDB_TYPE_BY_ATHENA = {
}
+# Readers that accept ``hive_types``, so partition columns get their Glue types.
+_HIVE_TYPED_READERS = {"parquet": "read_parquet", "csv": "read_csv", "json": "read_json"}
+
+
+def _file_relation(pattern, table_data):
+ """The table's files; partition keys are typed from Glue instead of inferred from the path."""
+ path = "'" + pattern.replace("'", "''") + "'"
+ reader = _HIVE_TYPED_READERS.get(_data_format(table_data))
+ partition_keys = table_data.get("PartitionKeys") or []
+ if not reader or not partition_keys:
+ return path # DuckDB reads the files, or reports why not
+ hive_types = ", ".join(
+ "'{}': {}".format(str(key.get("Name", "")).replace("'", "''"), _duckdb_type(key)) for key in partition_keys
+ )
+ return f"{reader}({path}, hive_partitioning = true, hive_types = {{{hive_types}}})"
+
+
+def _duckdb_type(column):
+ base = str(column.get("Type", "string")).lower().split("(")[0].strip()
+ return "DECIMAL" if base == "decimal" else _DUCKDB_TYPE_BY_ATHENA.get(base, "VARCHAR")
+
+
+def _qualified_column_prefixes(query, database, table, table_edit):
+ """Edits turning ``database.table.column`` (optionally catalog-qualified) into ``"table".column``.
+
+ The table itself is replaced by a relation aliased ``"table"``, so DuckDB cannot
+ resolve column references qualified with the database name.
+ """
+ tokens = list(_sql_tokens(query))
+ edits = []
+ for i in range(len(tokens) - 4):
+ if table_edit[0] <= tokens[i][2] < table_edit[1]:
+ continue # the FROM/JOIN reference itself
+ if i and tokens[i - 1][:2] == ("punct", "."):
+ continue
+ names, j = [], i
+ while j < len(tokens) and tokens[j][0] in ("name", "qname"):
+ names.append(tokens[j])
+ if j + 1 < len(tokens) and tokens[j + 1][:2] == ("punct", "."):
+ j += 2
+ else:
+ break
+ if len(names) not in (3, 4):
+ continue
+ db_token, table_token = names[-3], names[-2]
+ if db_token[1] == database and table_token[1] == table:
+ edits.append((names[0][2], table_token[3], '"{}"'.format(table.replace('"', '""'))))
+ return edits
+
+
def _empty_relation(table_data):
"""Zero rows with the table's Glue columns and partition keys; complex types read as VARCHAR."""
columns = ((table_data.get("StorageDescriptor") or {}).get("Columns") or []) + (table_data.get("PartitionKeys") or [])
@@ -1003,10 +1056,8 @@ def _empty_relation(table_data):
return "(SELECT 1 WHERE FALSE)"
projected = []
for column in columns:
- base = str(column.get("Type", "string")).lower().split("(")[0].strip()
- duck_type = "DECIMAL" if base == "decimal" else _DUCKDB_TYPE_BY_ATHENA.get(base, "VARCHAR")
name = str(column.get("Name", "")).replace('"', '""')
- projected.append(f'CAST(NULL AS {duck_type}) AS "{name}"')
+ projected.append(f'CAST(NULL AS {_duckdb_type(column)}) AS "{name}"')
return f"(SELECT {', '.join(projected)} WHERE FALSE)"
diff --git a/tests/test_athena.py b/tests/test_athena.py
index 38d6790d..098ebda4 100644
--- a/tests/test_athena.py
+++ b/tests/test_athena.py
@@ -1082,6 +1082,43 @@ def test_athena_reads_a_parquet_table_created_without_classification(
assert [row["Data"][0]["VarCharValue"] for row in rows] == ["id", "1", "2"]
+def test_athena_partitioned_table_types_partitions_and_resolves_qualified_columns(
+ athena, glue, s3, persisted_parquet_store, tmp_path,
+):
+ """Partition keys take their Glue types, and ``db.table.column`` resolves against the table."""
+ suffix = _uuid_mod.uuid4().hex[:10]
+ bucket, database = f"athena-partitions-{suffix}", f"partitions_{suffix}"
+ s3.create_bucket(Bucket=bucket)
+ glue.create_database(DatabaseInput={"Name": database})
+ parquet = tmp_path / "event.parquet"
+ with duckdb.connect() as connection:
+ connection.execute("COPY (SELECT 'a' AS note, 1 AS action) TO '%s' (FORMAT PARQUET)" % parquet)
+ persisted_parquet_store._persist_object(
+ bucket, "usage/year=2026/month=04/day=23/event.parquet", parquet.read_bytes(),
+ )
+ glue.create_table(DatabaseName=database, TableInput={
+ "Name": "usage",
+ "StorageDescriptor": {
+ "Location": f"s3://{bucket}/usage/",
+ "Columns": [{"Name": "note", "Type": "string"}, {"Name": "action", "Type": "int"}],
+ },
+ "PartitionKeys": [
+ {"Name": "year", "Type": "string"},
+ {"Name": "month", "Type": "string"},
+ {"Name": "day", "Type": "string"},
+ ],
+ "Parameters": {"classification": "parquet"},
+ })
+
+ query_id, execution = _run_to_completion(
+ athena, f"SELECT {database}.usage.note, year, month, day FROM {database}.usage", database,
+ )
+ assert execution["Status"]["State"] == "SUCCEEDED", execution["Status"]
+ result = athena.get_query_results(QueryExecutionId=query_id)["ResultSet"]
+ assert [cell["VarCharValue"] for cell in result["Rows"][1]["Data"]] == ["a", "2026", "04", "23"]
+ assert [column["Type"] for column in result["ResultSetMetadata"]["ColumnInfo"]] == ["varchar"] * 4
+
+
# ---- DDL and table-reference parsing (in-process, no server) ----