Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

feat(glue): add ColumnStatistics API family (Table + Partition) (#882)

Jonathan Meersonfffb067e8cf8
diff --git a/ministack/services/glue.py b/ministack/services/glue.py
index b5ad2a8b..89eeed74 100644
--- a/ministack/services/glue.py
+++ b/ministack/services/glue.py
@@ -112,6 +112,8 @@ _triggers = AccountScopedDict() # trigger_name -> trigger dict
_workflows = AccountScopedDict() # workflow_name -> workflow dict
_workflow_runs = AccountScopedDict() # workflow_name -> [run, ...]
_user_defined_functions = AccountScopedDict() # "db_name/function_name" -> udf dict
+_table_column_statistics = AccountScopedDict() # "db/table" -> {column_name: stats}
+_partition_column_statistics = AccountScopedDict() # "db/table" -> [{"Values": [...], "Stats": {col: stats}}]
_ALL_STATE = {
"databases": _databases,
@@ -129,6 +131,8 @@ _ALL_STATE = {
"workflows": _workflows,
"workflow_runs": _workflow_runs,
"user_defined_functions": _user_defined_functions,
+ "table_column_statistics": _table_column_statistics,
+ "partition_column_statistics": _partition_column_statistics,
}
@@ -200,6 +204,13 @@ async def handle_request(method, path, headers, body, query_params):
# Partition Indexes
"CreatePartitionIndex": _create_partition_index,
"GetPartitionIndexes": _get_partition_indexes,
+ # Column Statistics
+ "UpdateColumnStatisticsForTable": _update_column_statistics_for_table,
+ "GetColumnStatisticsForTable": _get_column_statistics_for_table,
+ "DeleteColumnStatisticsForTable": _delete_column_statistics_for_table,
+ "UpdateColumnStatisticsForPartition": _update_column_statistics_for_partition,
+ "GetColumnStatisticsForPartition": _get_column_statistics_for_partition,
+ "DeleteColumnStatisticsForPartition": _delete_column_statistics_for_partition,
# Connections
"CreateConnection": _create_connection,
"DeleteConnection": _delete_connection,
@@ -503,6 +514,8 @@ def _delete_database(data):
del _tables[k]
_partitions.pop(k, None)
_partition_indexes.pop(k, None)
+ _table_column_statistics.pop(k, None)
+ _partition_column_statistics.pop(k, None)
return json_response({})
@@ -576,6 +589,8 @@ def _delete_table(data):
_tables.pop(key, None)
_partitions.pop(key, None)
_partition_indexes.pop(key, None)
+ _table_column_statistics.pop(key, None)
+ _partition_column_statistics.pop(key, None)
return json_response({})
@@ -643,6 +658,8 @@ def _batch_delete_table(data):
del _tables[key]
_partitions.pop(key, None)
_partition_indexes.pop(key, None)
+ _table_column_statistics.pop(key, None)
+ _partition_column_statistics.pop(key, None)
return json_response({"Errors": errors})
@@ -680,6 +697,10 @@ def _delete_partition(data):
key = f"{db_name}/{table_name}"
if key in _partitions:
_partitions[key] = [p for p in _partitions[key] if p.get("Values") != values]
+ if key in _partition_column_statistics:
+ _partition_column_statistics[key] = [
+ e for e in _partition_column_statistics[key] if e.get("Values") != values
+ ]
return json_response({})
@@ -809,6 +830,134 @@ def _get_partition_indexes(data):
return json_response({"PartitionIndexDescriptorList": _partition_indexes.get(key, [])})
+# ---- Column Statistics ----
+
+def _partition_stats_entry(key, values):
+ for entry in _partition_column_statistics.get(key, []):
+ if entry.get("Values") == values:
+ return entry
+ return None
+
+
+def _update_column_statistics_for_table(data):
+ db_name = data.get("DatabaseName")
+ table_name = data.get("TableName")
+ key = f"{db_name}/{table_name}"
+ if key not in _tables:
+ return error_response_json("EntityNotFoundException",
+ f"Table {table_name} not found in {db_name}", 400)
+ stats_list = data.get("ColumnStatisticsList", [])
+ bucket = _table_column_statistics.setdefault(key, {})
+ errors = []
+ for cs in stats_list:
+ col = cs.get("ColumnName")
+ if not col:
+ errors.append({"ColumnStatistics": cs, "Error": {
+ "ErrorCode": "InvalidInputException",
+ "ErrorMessage": "ColumnName is required"}})
+ continue
+ bucket[col] = cs
+ return json_response({"Errors": errors})
+
+
+def _get_column_statistics_for_table(data):
+ db_name = data.get("DatabaseName")
+ table_name = data.get("TableName")
+ key = f"{db_name}/{table_name}"
+ if key not in _tables:
+ return error_response_json("EntityNotFoundException",
+ f"Table {table_name} not found in {db_name}", 400)
+ bucket = _table_column_statistics.get(key, {})
+ stats_list = []
+ errors = []
+ for col in data.get("ColumnNames", []):
+ if col in bucket:
+ stats_list.append(bucket[col])
+ else:
+ errors.append({"ColumnName": col, "Error": {
+ "ErrorCode": "EntityNotFoundException",
+ "ErrorMessage": f"Column statistics for {col} not found"}})
+ return json_response({"ColumnStatisticsList": stats_list, "Errors": errors})
+
+
+def _delete_column_statistics_for_table(data):
+ db_name = data.get("DatabaseName")
+ table_name = data.get("TableName")
+ column_name = data.get("ColumnName")
+ key = f"{db_name}/{table_name}"
+ if key not in _tables:
+ return error_response_json("EntityNotFoundException",
+ f"Table {table_name} not found in {db_name}", 400)
+ # Real Glue's Delete* operations are idempotent — deleting stats for a
+ # column that never had any returns 200 with an empty body.
+ _table_column_statistics.get(key, {}).pop(column_name, None)
+ return json_response({})
+
+
+def _update_column_statistics_for_partition(data):
+ db_name = data.get("DatabaseName")
+ table_name = data.get("TableName")
+ values = data.get("PartitionValues", [])
+ key = f"{db_name}/{table_name}"
+ if not any(p.get("Values") == values for p in _partitions.get(key, [])):
+ return error_response_json("EntityNotFoundException",
+ f"Partition with values {values} not found", 400)
+ entry = _partition_stats_entry(key, values)
+ if entry is None:
+ entry = {"Values": values, "Stats": {}}
+ _partition_column_statistics.setdefault(key, []).append(entry)
+ errors = []
+ for cs in data.get("ColumnStatisticsList", []):
+ col = cs.get("ColumnName")
+ if not col:
+ errors.append({"ColumnStatistics": cs, "Error": {
+ "ErrorCode": "InvalidInputException",
+ "ErrorMessage": "ColumnName is required"}})
+ continue
+ entry["Stats"][col] = cs
+ return json_response({"Errors": errors})
+
+
+def _get_column_statistics_for_partition(data):
+ db_name = data.get("DatabaseName")
+ table_name = data.get("TableName")
+ values = data.get("PartitionValues", [])
+ key = f"{db_name}/{table_name}"
+ if key not in _tables:
+ return error_response_json("EntityNotFoundException",
+ f"Table {table_name} not found in {db_name}", 400)
+ if not any(p.get("Values") == values for p in _partitions.get(key, [])):
+ return error_response_json("EntityNotFoundException",
+ f"Partition with values {values} not found", 400)
+ entry = _partition_stats_entry(key, values)
+ bucket = entry["Stats"] if entry else {}
+ stats_list = []
+ errors = []
+ for col in data.get("ColumnNames", []):
+ if col in bucket:
+ stats_list.append(bucket[col])
+ else:
+ errors.append({"ColumnName": col, "Error": {
+ "ErrorCode": "EntityNotFoundException",
+ "ErrorMessage": f"Column statistics for {col} not found"}})
+ return json_response({"ColumnStatisticsList": stats_list, "Errors": errors})
+
+
+def _delete_column_statistics_for_partition(data):
+ db_name = data.get("DatabaseName")
+ table_name = data.get("TableName")
+ values = data.get("PartitionValues", [])
+ column_name = data.get("ColumnName")
+ key = f"{db_name}/{table_name}"
+ if not any(p.get("Values") == values for p in _partitions.get(key, [])):
+ return error_response_json("EntityNotFoundException",
+ f"Partition with values {values} not found", 400)
+ entry = _partition_stats_entry(key, values)
+ if entry is not None:
+ entry.get("Stats", {}).pop(column_name, None)
+ return json_response({})
+
+
# ---- Connections ----
def _create_connection(data):
@@ -1791,3 +1940,5 @@ def reset():
_triggers.clear()
_workflows.clear()
_workflow_runs.clear()
+ _table_column_statistics.clear()
+ _partition_column_statistics.clear()
diff --git a/tests/test_glue.py b/tests/test_glue.py
index 0230daee..b4a4e1dd 100644
--- a/tests/test_glue.py
+++ b/tests/test_glue.py
@@ -1378,3 +1378,174 @@ def test_glue_resolve_script_isolated_per_account(tmp_path, monkeypatch):
assert gluemod._resolve_script(uri) is None
finally:
respmod._request_account_id.reset(tok_other)
+
+
+# ---- Column Statistics ----
+
+def _make_int_column_stats(column_name):
+ return {
+ "ColumnName": column_name,
+ "ColumnType": "int",
+ "AnalyzedTime": 1700000000,
+ "StatisticsData": {
+ "Type": "LONG",
+ "LongColumnStatisticsData": {
+ "MinimumValue": 1,
+ "MaximumValue": 100,
+ "NumberOfNulls": 0,
+ "NumberOfDistinctValues": 50,
+ },
+ },
+ }
+
+
+def _setup_stats_table(glue, db, table, partitioned=False):
+ glue.create_database(DatabaseInput={"Name": db})
+ table_input = {
+ "Name": table,
+ "StorageDescriptor": {
+ "Columns": [
+ {"Name": "id", "Type": "int"},
+ {"Name": "amount", "Type": "int"},
+ ],
+ "Location": f"s3://bucket/{db}/{table}/",
+ "InputFormat": "",
+ "OutputFormat": "",
+ "SerdeInfo": {},
+ },
+ }
+ if partitioned:
+ table_input["PartitionKeys"] = [{"Name": "dt", "Type": "string"}]
+ glue.create_table(DatabaseName=db, TableInput=table_input)
+
+
+def test_glue_column_statistics_for_table_crud(glue):
+ db, table = "qa-glue-colstats-tbl-db", "qa-glue-colstats-tbl"
+ _setup_stats_table(glue, db, table)
+
+ stats_id = _make_int_column_stats("id")
+ stats_amount = _make_int_column_stats("amount")
+ upd = glue.update_column_statistics_for_table(
+ DatabaseName=db, TableName=table,
+ ColumnStatisticsList=[stats_id, stats_amount],
+ )
+ assert upd.get("Errors", []) == []
+
+ got = glue.get_column_statistics_for_table(
+ DatabaseName=db, TableName=table, ColumnNames=["id", "amount"],
+ )
+ assert {s["ColumnName"] for s in got["ColumnStatisticsList"]} == {"id", "amount"}
+ assert got.get("Errors", []) == []
+
+ missing = glue.get_column_statistics_for_table(
+ DatabaseName=db, TableName=table, ColumnNames=["id", "missing"],
+ )
+ assert [s["ColumnName"] for s in missing["ColumnStatisticsList"]] == ["id"]
+ assert missing["Errors"][0]["ColumnName"] == "missing"
+
+ glue.delete_column_statistics_for_table(
+ DatabaseName=db, TableName=table, ColumnName="id",
+ )
+ after = glue.get_column_statistics_for_table(
+ DatabaseName=db, TableName=table, ColumnNames=["id", "amount"],
+ )
+ assert [s["ColumnName"] for s in after["ColumnStatisticsList"]] == ["amount"]
+
+
+def test_glue_column_statistics_for_table_missing_table(glue):
+ with pytest.raises(ClientError) as exc:
+ glue.get_column_statistics_for_table(
+ DatabaseName="nope-db", TableName="nope-tbl", ColumnNames=["id"],
+ )
+ assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"
+
+
+def test_glue_delete_column_statistics_for_table_unknown_column_is_idempotent(glue):
+ db, table = "qa-glue-colstats-tbl-del-db", "qa-glue-colstats-tbl-del"
+ _setup_stats_table(glue, db, table)
+ # Real AWS Glue returns 200 / empty body when deleting stats for a column
+ # that never had any — Delete* operations are idempotent.
+ glue.delete_column_statistics_for_table(
+ DatabaseName=db, TableName=table, ColumnName="never-set",
+ )
+
+
+def test_glue_column_statistics_cleared_on_table_delete(glue):
+ db, table = "qa-glue-colstats-cleanup-db", "qa-glue-colstats-cleanup"
+ _setup_stats_table(glue, db, table)
+ glue.update_column_statistics_for_table(
+ DatabaseName=db, TableName=table,
+ ColumnStatisticsList=[_make_int_column_stats("id")],
+ )
+ glue.delete_table(DatabaseName=db, Name=table)
+ # Re-create the table with the same name; stats from the previous
+ # incarnation must not leak through.
+ glue.create_table(
+ DatabaseName=db,
+ TableInput={
+ "Name": table,
+ "StorageDescriptor": {
+ "Columns": [{"Name": "id", "Type": "int"}],
+ "Location": f"s3://bucket/{db}/{table}/",
+ "InputFormat": "", "OutputFormat": "", "SerdeInfo": {},
+ },
+ },
+ )
+ got = glue.get_column_statistics_for_table(
+ DatabaseName=db, TableName=table, ColumnNames=["id"],
+ )
+ assert got["ColumnStatisticsList"] == []
+ assert got["Errors"][0]["ColumnName"] == "id"
+
+
+def test_glue_column_statistics_for_partition_crud(glue):
+ db, table = "qa-glue-colstats-part-db", "qa-glue-colstats-part"
+ _setup_stats_table(glue, db, table, partitioned=True)
+ glue.create_partition(
+ DatabaseName=db, TableName=table,
+ PartitionInput={
+ "Values": ["2024-01-01"],
+ "StorageDescriptor": {
+ "Columns": [], "Location": f"s3://bucket/{db}/{table}/dt=2024-01-01",
+ "InputFormat": "", "OutputFormat": "", "SerdeInfo": {},
+ },
+ },
+ )
+
+ stats_id = _make_int_column_stats("id")
+ stats_amount = _make_int_column_stats("amount")
+ upd = glue.update_column_statistics_for_partition(
+ DatabaseName=db, TableName=table,
+ PartitionValues=["2024-01-01"],
+ ColumnStatisticsList=[stats_id, stats_amount],
+ )
+ assert upd.get("Errors", []) == []
+
+ got = glue.get_column_statistics_for_partition(
+ DatabaseName=db, TableName=table,
+ PartitionValues=["2024-01-01"], ColumnNames=["id", "amount"],
+ )
+ assert {s["ColumnName"] for s in got["ColumnStatisticsList"]} == {"id", "amount"}
+
+ glue.delete_column_statistics_for_partition(
+ DatabaseName=db, TableName=table,
+ PartitionValues=["2024-01-01"], ColumnName="amount",
+ )
+ after = glue.get_column_statistics_for_partition(
+ DatabaseName=db, TableName=table,
+ PartitionValues=["2024-01-01"], ColumnNames=["id", "amount"],
+ )
+ assert [s["ColumnName"] for s in after["ColumnStatisticsList"]] == ["id"]
+ assert after["Errors"][0]["ColumnName"] == "amount"
+
+
+def test_glue_column_statistics_for_partition_missing_partition(glue):
+ db, table = "qa-glue-colstats-nopart-db", "qa-glue-colstats-nopart"
+ _setup_stats_table(glue, db, table, partitioned=True)
+ with pytest.raises(ClientError) as exc:
+ glue.update_column_statistics_for_partition(
+ DatabaseName=db, TableName=table,
+ PartitionValues=["never"],
+ ColumnStatisticsList=[_make_int_column_stats("id")],
+ )
+ assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"