Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
feat(glue): add ColumnStatistics API family (Table + Partition) (#882)
Jonathan Meersonfffb067e8cf8
diff --git a/ministack/services/glue.py b/ministack/services/glue.pyindex b5ad2a8b..89eeed74 100644--- a/ministack/services/glue.py+++ b/ministack/services/glue.py@@ -112,6 +112,8 @@ _triggers = AccountScopedDict() # trigger_name -> trigger dict_workflows = AccountScopedDict() # workflow_name -> workflow dict_workflow_runs = AccountScopedDict() # workflow_name -> [run, ...]_user_defined_functions = AccountScopedDict() # "db_name/function_name" -> udf dict+_table_column_statistics = AccountScopedDict() # "db/table" -> {column_name: stats}+_partition_column_statistics = AccountScopedDict() # "db/table" -> [{"Values": [...], "Stats": {col: stats}}]_ALL_STATE = {"databases": _databases,@@ -129,6 +131,8 @@ _ALL_STATE = {"workflows": _workflows,"workflow_runs": _workflow_runs,"user_defined_functions": _user_defined_functions,+ "table_column_statistics": _table_column_statistics,+ "partition_column_statistics": _partition_column_statistics,}@@ -200,6 +204,13 @@ async def handle_request(method, path, headers, body, query_params):# Partition Indexes"CreatePartitionIndex": _create_partition_index,"GetPartitionIndexes": _get_partition_indexes,+ # Column Statistics+ "UpdateColumnStatisticsForTable": _update_column_statistics_for_table,+ "GetColumnStatisticsForTable": _get_column_statistics_for_table,+ "DeleteColumnStatisticsForTable": _delete_column_statistics_for_table,+ "UpdateColumnStatisticsForPartition": _update_column_statistics_for_partition,+ "GetColumnStatisticsForPartition": _get_column_statistics_for_partition,+ "DeleteColumnStatisticsForPartition": _delete_column_statistics_for_partition,# Connections"CreateConnection": _create_connection,"DeleteConnection": _delete_connection,@@ -503,6 +514,8 @@ def _delete_database(data):del _tables[k]_partitions.pop(k, None)_partition_indexes.pop(k, None)+ _table_column_statistics.pop(k, None)+ _partition_column_statistics.pop(k, None)return json_response({})@@ -576,6 +589,8 @@ def _delete_table(data):_tables.pop(key, None)_partitions.pop(key, None)_partition_indexes.pop(key, None)+ _table_column_statistics.pop(key, None)+ _partition_column_statistics.pop(key, None)return json_response({})@@ -643,6 +658,8 @@ def _batch_delete_table(data):del _tables[key]_partitions.pop(key, None)_partition_indexes.pop(key, None)+ _table_column_statistics.pop(key, None)+ _partition_column_statistics.pop(key, None)return json_response({"Errors": errors})@@ -680,6 +697,10 @@ def _delete_partition(data):key = f"{db_name}/{table_name}"if key in _partitions:_partitions[key] = [p for p in _partitions[key] if p.get("Values") != values]+ if key in _partition_column_statistics:+ _partition_column_statistics[key] = [+ e for e in _partition_column_statistics[key] if e.get("Values") != values+ ]return json_response({})@@ -809,6 +830,134 @@ def _get_partition_indexes(data):return json_response({"PartitionIndexDescriptorList": _partition_indexes.get(key, [])})+# ---- Column Statistics ----++def _partition_stats_entry(key, values):+ for entry in _partition_column_statistics.get(key, []):+ if entry.get("Values") == values:+ return entry+ return None+++def _update_column_statistics_for_table(data):+ db_name = data.get("DatabaseName")+ table_name = data.get("TableName")+ key = f"{db_name}/{table_name}"+ if key not in _tables:+ return error_response_json("EntityNotFoundException",+ f"Table {table_name} not found in {db_name}", 400)+ stats_list = data.get("ColumnStatisticsList", [])+ bucket = _table_column_statistics.setdefault(key, {})+ errors = []+ for cs in stats_list:+ col = cs.get("ColumnName")+ if not col:+ errors.append({"ColumnStatistics": cs, "Error": {+ "ErrorCode": "InvalidInputException",+ "ErrorMessage": "ColumnName is required"}})+ continue+ bucket[col] = cs+ return json_response({"Errors": errors})+++def _get_column_statistics_for_table(data):+ db_name = data.get("DatabaseName")+ table_name = data.get("TableName")+ key = f"{db_name}/{table_name}"+ if key not in _tables:+ return error_response_json("EntityNotFoundException",+ f"Table {table_name} not found in {db_name}", 400)+ bucket = _table_column_statistics.get(key, {})+ stats_list = []+ errors = []+ for col in data.get("ColumnNames", []):+ if col in bucket:+ stats_list.append(bucket[col])+ else:+ errors.append({"ColumnName": col, "Error": {+ "ErrorCode": "EntityNotFoundException",+ "ErrorMessage": f"Column statistics for {col} not found"}})+ return json_response({"ColumnStatisticsList": stats_list, "Errors": errors})+++def _delete_column_statistics_for_table(data):+ db_name = data.get("DatabaseName")+ table_name = data.get("TableName")+ column_name = data.get("ColumnName")+ key = f"{db_name}/{table_name}"+ if key not in _tables:+ return error_response_json("EntityNotFoundException",+ f"Table {table_name} not found in {db_name}", 400)+ # Real Glue's Delete* operations are idempotent — deleting stats for a+ # column that never had any returns 200 with an empty body.+ _table_column_statistics.get(key, {}).pop(column_name, None)+ return json_response({})+++def _update_column_statistics_for_partition(data):+ db_name = data.get("DatabaseName")+ table_name = data.get("TableName")+ values = data.get("PartitionValues", [])+ key = f"{db_name}/{table_name}"+ if not any(p.get("Values") == values for p in _partitions.get(key, [])):+ return error_response_json("EntityNotFoundException",+ f"Partition with values {values} not found", 400)+ entry = _partition_stats_entry(key, values)+ if entry is None:+ entry = {"Values": values, "Stats": {}}+ _partition_column_statistics.setdefault(key, []).append(entry)+ errors = []+ for cs in data.get("ColumnStatisticsList", []):+ col = cs.get("ColumnName")+ if not col:+ errors.append({"ColumnStatistics": cs, "Error": {+ "ErrorCode": "InvalidInputException",+ "ErrorMessage": "ColumnName is required"}})+ continue+ entry["Stats"][col] = cs+ return json_response({"Errors": errors})+++def _get_column_statistics_for_partition(data):+ db_name = data.get("DatabaseName")+ table_name = data.get("TableName")+ values = data.get("PartitionValues", [])+ key = f"{db_name}/{table_name}"+ if key not in _tables:+ return error_response_json("EntityNotFoundException",+ f"Table {table_name} not found in {db_name}", 400)+ if not any(p.get("Values") == values for p in _partitions.get(key, [])):+ return error_response_json("EntityNotFoundException",+ f"Partition with values {values} not found", 400)+ entry = _partition_stats_entry(key, values)+ bucket = entry["Stats"] if entry else {}+ stats_list = []+ errors = []+ for col in data.get("ColumnNames", []):+ if col in bucket:+ stats_list.append(bucket[col])+ else:+ errors.append({"ColumnName": col, "Error": {+ "ErrorCode": "EntityNotFoundException",+ "ErrorMessage": f"Column statistics for {col} not found"}})+ return json_response({"ColumnStatisticsList": stats_list, "Errors": errors})+++def _delete_column_statistics_for_partition(data):+ db_name = data.get("DatabaseName")+ table_name = data.get("TableName")+ values = data.get("PartitionValues", [])+ column_name = data.get("ColumnName")+ key = f"{db_name}/{table_name}"+ if not any(p.get("Values") == values for p in _partitions.get(key, [])):+ return error_response_json("EntityNotFoundException",+ f"Partition with values {values} not found", 400)+ entry = _partition_stats_entry(key, values)+ if entry is not None:+ entry.get("Stats", {}).pop(column_name, None)+ return json_response({})++# ---- Connections ----def _create_connection(data):@@ -1791,3 +1940,5 @@ def reset():_triggers.clear()_workflows.clear()_workflow_runs.clear()+ _table_column_statistics.clear()+ _partition_column_statistics.clear()diff --git a/tests/test_glue.py b/tests/test_glue.pyindex 0230daee..b4a4e1dd 100644--- a/tests/test_glue.py+++ b/tests/test_glue.py@@ -1378,3 +1378,174 @@ def test_glue_resolve_script_isolated_per_account(tmp_path, monkeypatch):assert gluemod._resolve_script(uri) is Nonefinally:respmod._request_account_id.reset(tok_other)+++# ---- Column Statistics ----++def _make_int_column_stats(column_name):+ return {+ "ColumnName": column_name,+ "ColumnType": "int",+ "AnalyzedTime": 1700000000,+ "StatisticsData": {+ "Type": "LONG",+ "LongColumnStatisticsData": {+ "MinimumValue": 1,+ "MaximumValue": 100,+ "NumberOfNulls": 0,+ "NumberOfDistinctValues": 50,+ },+ },+ }+++def _setup_stats_table(glue, db, table, partitioned=False):+ glue.create_database(DatabaseInput={"Name": db})+ table_input = {+ "Name": table,+ "StorageDescriptor": {+ "Columns": [+ {"Name": "id", "Type": "int"},+ {"Name": "amount", "Type": "int"},+ ],+ "Location": f"s3://bucket/{db}/{table}/",+ "InputFormat": "",+ "OutputFormat": "",+ "SerdeInfo": {},+ },+ }+ if partitioned:+ table_input["PartitionKeys"] = [{"Name": "dt", "Type": "string"}]+ glue.create_table(DatabaseName=db, TableInput=table_input)+++def test_glue_column_statistics_for_table_crud(glue):+ db, table = "qa-glue-colstats-tbl-db", "qa-glue-colstats-tbl"+ _setup_stats_table(glue, db, table)++ stats_id = _make_int_column_stats("id")+ stats_amount = _make_int_column_stats("amount")+ upd = glue.update_column_statistics_for_table(+ DatabaseName=db, TableName=table,+ ColumnStatisticsList=[stats_id, stats_amount],+ )+ assert upd.get("Errors", []) == []++ got = glue.get_column_statistics_for_table(+ DatabaseName=db, TableName=table, ColumnNames=["id", "amount"],+ )+ assert {s["ColumnName"] for s in got["ColumnStatisticsList"]} == {"id", "amount"}+ assert got.get("Errors", []) == []++ missing = glue.get_column_statistics_for_table(+ DatabaseName=db, TableName=table, ColumnNames=["id", "missing"],+ )+ assert [s["ColumnName"] for s in missing["ColumnStatisticsList"]] == ["id"]+ assert missing["Errors"][0]["ColumnName"] == "missing"++ glue.delete_column_statistics_for_table(+ DatabaseName=db, TableName=table, ColumnName="id",+ )+ after = glue.get_column_statistics_for_table(+ DatabaseName=db, TableName=table, ColumnNames=["id", "amount"],+ )+ assert [s["ColumnName"] for s in after["ColumnStatisticsList"]] == ["amount"]+++def test_glue_column_statistics_for_table_missing_table(glue):+ with pytest.raises(ClientError) as exc:+ glue.get_column_statistics_for_table(+ DatabaseName="nope-db", TableName="nope-tbl", ColumnNames=["id"],+ )+ assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"+++def test_glue_delete_column_statistics_for_table_unknown_column_is_idempotent(glue):+ db, table = "qa-glue-colstats-tbl-del-db", "qa-glue-colstats-tbl-del"+ _setup_stats_table(glue, db, table)+ # Real AWS Glue returns 200 / empty body when deleting stats for a column+ # that never had any — Delete* operations are idempotent.+ glue.delete_column_statistics_for_table(+ DatabaseName=db, TableName=table, ColumnName="never-set",+ )+++def test_glue_column_statistics_cleared_on_table_delete(glue):+ db, table = "qa-glue-colstats-cleanup-db", "qa-glue-colstats-cleanup"+ _setup_stats_table(glue, db, table)+ glue.update_column_statistics_for_table(+ DatabaseName=db, TableName=table,+ ColumnStatisticsList=[_make_int_column_stats("id")],+ )+ glue.delete_table(DatabaseName=db, Name=table)+ # Re-create the table with the same name; stats from the previous+ # incarnation must not leak through.+ glue.create_table(+ DatabaseName=db,+ TableInput={+ "Name": table,+ "StorageDescriptor": {+ "Columns": [{"Name": "id", "Type": "int"}],+ "Location": f"s3://bucket/{db}/{table}/",+ "InputFormat": "", "OutputFormat": "", "SerdeInfo": {},+ },+ },+ )+ got = glue.get_column_statistics_for_table(+ DatabaseName=db, TableName=table, ColumnNames=["id"],+ )+ assert got["ColumnStatisticsList"] == []+ assert got["Errors"][0]["ColumnName"] == "id"+++def test_glue_column_statistics_for_partition_crud(glue):+ db, table = "qa-glue-colstats-part-db", "qa-glue-colstats-part"+ _setup_stats_table(glue, db, table, partitioned=True)+ glue.create_partition(+ DatabaseName=db, TableName=table,+ PartitionInput={+ "Values": ["2024-01-01"],+ "StorageDescriptor": {+ "Columns": [], "Location": f"s3://bucket/{db}/{table}/dt=2024-01-01",+ "InputFormat": "", "OutputFormat": "", "SerdeInfo": {},+ },+ },+ )++ stats_id = _make_int_column_stats("id")+ stats_amount = _make_int_column_stats("amount")+ upd = glue.update_column_statistics_for_partition(+ DatabaseName=db, TableName=table,+ PartitionValues=["2024-01-01"],+ ColumnStatisticsList=[stats_id, stats_amount],+ )+ assert upd.get("Errors", []) == []++ got = glue.get_column_statistics_for_partition(+ DatabaseName=db, TableName=table,+ PartitionValues=["2024-01-01"], ColumnNames=["id", "amount"],+ )+ assert {s["ColumnName"] for s in got["ColumnStatisticsList"]} == {"id", "amount"}++ glue.delete_column_statistics_for_partition(+ DatabaseName=db, TableName=table,+ PartitionValues=["2024-01-01"], ColumnName="amount",+ )+ after = glue.get_column_statistics_for_partition(+ DatabaseName=db, TableName=table,+ PartitionValues=["2024-01-01"], ColumnNames=["id", "amount"],+ )+ assert [s["ColumnName"] for s in after["ColumnStatisticsList"]] == ["id"]+ assert after["Errors"][0]["ColumnName"] == "amount"+++def test_glue_column_statistics_for_partition_missing_partition(glue):+ db, table = "qa-glue-colstats-nopart-db", "qa-glue-colstats-nopart"+ _setup_stats_table(glue, db, table, partitioned=True)+ with pytest.raises(ClientError) as exc:+ glue.update_column_statistics_for_partition(+ DatabaseName=db, TableName=table,+ PartitionValues=["never"],+ ColumnStatisticsList=[_make_int_column_stats("id")],+ )+ assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"