Light, fluffy, and always free - The AWS Local Emulator alternative
test: add Glue catalog compatibility tests (#1222)
David Phillips8bef8820b2de
diff --git a/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats b/compatibility-tests/sdk-test-awscli/test/glue-catalog.batsnew file mode 100644index 0000000000..ed823bf218--- /dev/null+++ b/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats@@ -0,0 +1,211 @@+#!/usr/bin/env bats++setup() {+ load 'test_helper/common-setup'+ DB_NAME="$(unique_name glue-catalog-db)"+ TABLE_NAME="catalog_table"+ FUNCTION_NAME="catalog_function"+}++teardown() {+ aws_cmd glue delete-user-defined-function \+ --database-name "$DB_NAME" \+ --function-name "$FUNCTION_NAME" >/dev/null 2>&1 || true+ aws_cmd glue delete-table \+ --database-name "$DB_NAME" \+ --name "$TABLE_NAME" >/dev/null 2>&1 || true+ aws_cmd glue delete-database \+ --name "$DB_NAME" >/dev/null 2>&1 || true+}++create_database() {+ aws_cmd glue create-database \+ --database-input "{\"Name\":\"$DB_NAME\",\"Description\":\"catalog compatibility database\"}" >/dev/null+}++table_input() {+ local description="$1"+ jq -n \+ --arg name "$TABLE_NAME" \+ --arg description "$description" \+ --arg location "s3://floci-glue-catalog/$DB_NAME/$TABLE_NAME/" \+ '{+ Name: $name,+ Description: $description,+ Parameters: {+ classification: "json"+ },+ StorageDescriptor: {+ Location: $location,+ InputFormat: "org.apache.hadoop.mapred.TextInputFormat",+ OutputFormat: "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",+ SerdeInfo: {+ SerializationLibrary: "org.openx.data.jsonserde.JsonSerDe",+ Parameters: {+ "serialization.format": "1"+ }+ },+ Columns: [+ {+ Name: "id",+ Type: "int"+ }+ ]+ }+ }'+}++create_table() {+ aws_cmd glue create-table \+ --database-name "$DB_NAME" \+ --table-input "$(table_input created)" >/dev/null+}++function_input() {+ local owner="$1"+ jq -n \+ --arg name "$FUNCTION_NAME" \+ --arg owner "$owner" \+ '{+ FunctionName: $name,+ ClassName: "CatalogFunction",+ OwnerName: $owner,+ OwnerType: "USER",+ ResourceUris: [+ {+ ResourceType: "FILE",+ Uri: "s3://floci-glue-catalog/function.jar"+ }+ ]+ }'+}++@test "Glue catalog: database lifecycle" {+ run create_database+ assert_success++ run aws_cmd glue get-database --name "$DB_NAME"+ assert_success+ name=$(json_get "$output" '.Database.Name')+ [ "$name" = "$DB_NAME" ]++ run aws_cmd glue get-databases+ assert_success+ found=$(echo "$output" | jq --arg name "$DB_NAME" '.DatabaseList | any(.Name == $name)')+ [ "$found" = "true" ]++ run aws_cmd glue delete-database --name "$DB_NAME"+ assert_success++ run aws_cmd glue get-database --name "$DB_NAME"+ assert_failure+}++@test "Glue catalog: table and partition lifecycle" {+ run create_database+ assert_success++ run create_table+ assert_success++ run aws_cmd glue get-table \+ --database-name "$DB_NAME" \+ --name "$TABLE_NAME"+ assert_success+ name=$(json_get "$output" '.Table.Name')+ version_id=$(json_get "$output" '.Table.VersionId')+ [ "$name" = "$TABLE_NAME" ]+ [ "$version_id" = "0" ]++ run aws_cmd glue get-tables --database-name "$DB_NAME"+ assert_success+ found=$(echo "$output" | jq --arg name "$TABLE_NAME" '.TableList | any(.Name == $name)')+ [ "$found" = "true" ]++ run aws_cmd glue update-table \+ --database-name "$DB_NAME" \+ --version-id "$version_id" \+ --table-input "$(table_input updated)"+ assert_success++ run aws_cmd glue get-table \+ --database-name "$DB_NAME" \+ --name "$TABLE_NAME"+ assert_success+ description=$(json_get "$output" '.Table.Description')+ version_id=$(json_get "$output" '.Table.VersionId')+ [ "$description" = "updated" ]+ [ "$version_id" = "1" ]++ run aws_cmd glue create-partition \+ --database-name "$DB_NAME" \+ --table-name "$TABLE_NAME" \+ --partition-input '{"Values":["2026"]}'+ assert_success++ run aws_cmd glue get-partitions \+ --database-name "$DB_NAME" \+ --table-name "$TABLE_NAME"+ assert_success+ value=$(json_get "$output" '.Partitions[0].Values[0]')+ [ "$value" = "2026" ]++ run aws_cmd glue delete-table \+ --database-name "$DB_NAME" \+ --name "$TABLE_NAME"+ assert_success++ run aws_cmd glue get-table \+ --database-name "$DB_NAME" \+ --name "$TABLE_NAME"+ assert_failure+}++@test "Glue catalog: user-defined function lifecycle" {+ run create_database+ assert_success++ run aws_cmd glue create-user-defined-function \+ --database-name "$DB_NAME" \+ --function-input "$(function_input created-owner)"+ assert_success++ run aws_cmd glue get-user-defined-function \+ --database-name "$DB_NAME" \+ --function-name "$FUNCTION_NAME"+ assert_success+ name=$(json_get "$output" '.UserDefinedFunction.FunctionName')+ owner=$(json_get "$output" '.UserDefinedFunction.OwnerName')+ [ "$name" = "$FUNCTION_NAME" ]+ [ "$owner" = "created-owner" ]++ run aws_cmd glue get-user-defined-functions \+ --database-name "$DB_NAME" \+ --pattern "catalog_.*"+ assert_success+ found=$(echo "$output" | jq --arg name "$FUNCTION_NAME" '.UserDefinedFunctions | any(.FunctionName == $name)')+ [ "$found" = "true" ]++ run aws_cmd glue update-user-defined-function \+ --database-name "$DB_NAME" \+ --function-name "$FUNCTION_NAME" \+ --function-input "$(function_input updated-owner)"+ assert_success++ run aws_cmd glue get-user-defined-function \+ --database-name "$DB_NAME" \+ --function-name "$FUNCTION_NAME"+ assert_success+ owner=$(json_get "$output" '.UserDefinedFunction.OwnerName')+ [ "$owner" = "updated-owner" ]++ run aws_cmd glue delete-user-defined-function \+ --database-name "$DB_NAME" \+ --function-name "$FUNCTION_NAME"+ assert_success++ run aws_cmd glue get-user-defined-function \+ --database-name "$DB_NAME" \+ --function-name "$FUNCTION_NAME"+ assert_failure+}diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.javanew file mode 100644index 0000000000..ec138841f2--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/GlueCatalogTest.java@@ -0,0 +1,240 @@+package com.floci.test;++import org.junit.jupiter.api.AfterAll;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.Test;+import software.amazon.awssdk.services.glue.GlueClient;+import software.amazon.awssdk.services.glue.model.Column;+import software.amazon.awssdk.services.glue.model.CreateDatabaseRequest;+import software.amazon.awssdk.services.glue.model.CreatePartitionRequest;+import software.amazon.awssdk.services.glue.model.CreateTableRequest;+import software.amazon.awssdk.services.glue.model.CreateUserDefinedFunctionRequest;+import software.amazon.awssdk.services.glue.model.DatabaseInput;+import software.amazon.awssdk.services.glue.model.DeleteDatabaseRequest;+import software.amazon.awssdk.services.glue.model.DeleteTableRequest;+import software.amazon.awssdk.services.glue.model.DeleteUserDefinedFunctionRequest;+import software.amazon.awssdk.services.glue.model.EntityNotFoundException;+import software.amazon.awssdk.services.glue.model.GetDatabaseRequest;+import software.amazon.awssdk.services.glue.model.GetDatabasesRequest;+import software.amazon.awssdk.services.glue.model.GetPartitionsRequest;+import software.amazon.awssdk.services.glue.model.GetTableRequest;+import software.amazon.awssdk.services.glue.model.GetTablesRequest;+import software.amazon.awssdk.services.glue.model.GetUserDefinedFunctionRequest;+import software.amazon.awssdk.services.glue.model.GetUserDefinedFunctionsRequest;+import software.amazon.awssdk.services.glue.model.PartitionInput;+import software.amazon.awssdk.services.glue.model.PrincipalType;+import software.amazon.awssdk.services.glue.model.ResourceType;+import software.amazon.awssdk.services.glue.model.ResourceUri;+import software.amazon.awssdk.services.glue.model.SerDeInfo;+import software.amazon.awssdk.services.glue.model.StorageDescriptor;+import software.amazon.awssdk.services.glue.model.TableInput;+import software.amazon.awssdk.services.glue.model.UpdateTableRequest;+import software.amazon.awssdk.services.glue.model.UpdateUserDefinedFunctionRequest;+import software.amazon.awssdk.services.glue.model.UserDefinedFunction;+import software.amazon.awssdk.services.glue.model.UserDefinedFunctionInput;++import java.util.Map;++import static org.assertj.core.api.Assertions.assertThat;+import static org.assertj.core.api.Assertions.assertThatThrownBy;++@DisplayName("Glue catalog")+class GlueCatalogTest {++ private static final String DATABASE_NAME = TestFixtures.uniqueName("catalog_db");+ private static final String TABLE_NAME = "catalog_table";+ private static final String FUNCTION_NAME = "catalog_function";++ private static GlueClient glue;++ @BeforeAll+ static void setup() {+ glue = TestFixtures.glueClient();+ }++ @AfterAll+ static void cleanup() {+ if (glue == null) {+ return;+ }+ try {+ glue.deleteUserDefinedFunction(DeleteUserDefinedFunctionRequest.builder()+ .databaseName(DATABASE_NAME)+ .functionName(FUNCTION_NAME)+ .build());+ }+ catch (Exception ignored) {}+ try {+ glue.deleteTable(DeleteTableRequest.builder()+ .databaseName(DATABASE_NAME)+ .name(TABLE_NAME)+ .build());+ }+ catch (Exception ignored) {}+ try {+ glue.deleteDatabase(DeleteDatabaseRequest.builder()+ .name(DATABASE_NAME)+ .build());+ }+ catch (Exception ignored) {}+ glue.close();+ }++ @Test+ void catalogLifecycle() {+ glue.createDatabase(CreateDatabaseRequest.builder()+ .databaseInput(DatabaseInput.builder()+ .name(DATABASE_NAME)+ .description("catalog compatibility database")+ .build())+ .build());++ assertThat(glue.getDatabase(GetDatabaseRequest.builder()+ .name(DATABASE_NAME)+ .build()).database().name())+ .isEqualTo(DATABASE_NAME);+ assertThat(glue.getDatabases(GetDatabasesRequest.builder().build()).databaseList())+ .extracting(database -> database.name())+ .contains(DATABASE_NAME);++ glue.createTable(CreateTableRequest.builder()+ .databaseName(DATABASE_NAME)+ .tableInput(tableInput("created"))+ .build());++ var createdTable = glue.getTable(GetTableRequest.builder()+ .databaseName(DATABASE_NAME)+ .name(TABLE_NAME)+ .build()).table();+ assertThat(createdTable.databaseName()).isEqualTo(DATABASE_NAME);+ assertThat(createdTable.name()).isEqualTo(TABLE_NAME);+ assertThat(createdTable.versionId()).isEqualTo("0");+ assertThat(glue.getTables(GetTablesRequest.builder()+ .databaseName(DATABASE_NAME)+ .build()).tableList())+ .extracting(table -> table.name())+ .contains(TABLE_NAME);++ glue.updateTable(UpdateTableRequest.builder()+ .databaseName(DATABASE_NAME)+ .versionId(createdTable.versionId())+ .tableInput(tableInput("updated"))+ .build());+ var updatedTable = glue.getTable(GetTableRequest.builder()+ .databaseName(DATABASE_NAME)+ .name(TABLE_NAME)+ .build()).table();+ assertThat(updatedTable.description()).isEqualTo("updated");+ assertThat(updatedTable.versionId()).isEqualTo("1");++ glue.createPartition(CreatePartitionRequest.builder()+ .databaseName(DATABASE_NAME)+ .tableName(TABLE_NAME)+ .partitionInput(PartitionInput.builder()+ .values("2026")+ .build())+ .build());+ assertThat(glue.getPartitions(GetPartitionsRequest.builder()+ .databaseName(DATABASE_NAME)+ .tableName(TABLE_NAME)+ .build()).partitions())+ .singleElement()+ .satisfies(partition -> {+ assertThat(partition.databaseName()).isEqualTo(DATABASE_NAME);+ assertThat(partition.tableName()).isEqualTo(TABLE_NAME);+ assertThat(partition.values()).containsExactly("2026");+ });++ glue.createUserDefinedFunction(CreateUserDefinedFunctionRequest.builder()+ .databaseName(DATABASE_NAME)+ .functionInput(functionInput("created-owner"))+ .build());+ UserDefinedFunction createdFunction = glue.getUserDefinedFunction(GetUserDefinedFunctionRequest.builder()+ .databaseName(DATABASE_NAME)+ .functionName(FUNCTION_NAME)+ .build()).userDefinedFunction();+ assertThat(createdFunction.databaseName()).isEqualTo(DATABASE_NAME);+ assertThat(createdFunction.functionName()).isEqualTo(FUNCTION_NAME);+ assertThat(createdFunction.ownerName()).isEqualTo("created-owner");+ assertThat(glue.getUserDefinedFunctions(GetUserDefinedFunctionsRequest.builder()+ .databaseName(DATABASE_NAME)+ .pattern("catalog_.*")+ .build()).userDefinedFunctions())+ .extracting(UserDefinedFunction::functionName)+ .containsExactly(FUNCTION_NAME);++ glue.updateUserDefinedFunction(UpdateUserDefinedFunctionRequest.builder()+ .databaseName(DATABASE_NAME)+ .functionName(FUNCTION_NAME)+ .functionInput(functionInput("updated-owner"))+ .build());+ assertThat(glue.getUserDefinedFunction(GetUserDefinedFunctionRequest.builder()+ .databaseName(DATABASE_NAME)+ .functionName(FUNCTION_NAME)+ .build()).userDefinedFunction().ownerName())+ .isEqualTo("updated-owner");++ glue.deleteUserDefinedFunction(DeleteUserDefinedFunctionRequest.builder()+ .databaseName(DATABASE_NAME)+ .functionName(FUNCTION_NAME)+ .build());+ assertThatThrownBy(() -> glue.getUserDefinedFunction(GetUserDefinedFunctionRequest.builder()+ .databaseName(DATABASE_NAME)+ .functionName(FUNCTION_NAME)+ .build()))+ .isInstanceOf(EntityNotFoundException.class);++ glue.deleteTable(DeleteTableRequest.builder()+ .databaseName(DATABASE_NAME)+ .name(TABLE_NAME)+ .build());+ assertThatThrownBy(() -> glue.getTable(GetTableRequest.builder()+ .databaseName(DATABASE_NAME)+ .name(TABLE_NAME)+ .build()))+ .isInstanceOf(EntityNotFoundException.class);++ glue.deleteDatabase(DeleteDatabaseRequest.builder()+ .name(DATABASE_NAME)+ .build());+ assertThatThrownBy(() -> glue.getDatabase(GetDatabaseRequest.builder()+ .name(DATABASE_NAME)+ .build()))+ .isInstanceOf(EntityNotFoundException.class);+ }++ private static TableInput tableInput(String description) {+ return TableInput.builder()+ .name(TABLE_NAME)+ .description(description)+ .parameters(Map.of("classification", "json"))+ .storageDescriptor(StorageDescriptor.builder()+ .location("s3://floci-glue-catalog/" + DATABASE_NAME + "/" + TABLE_NAME + "/")+ .inputFormat("org.apache.hadoop.mapred.TextInputFormat")+ .outputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat")+ .serdeInfo(SerDeInfo.builder()+ .serializationLibrary("org.openx.data.jsonserde.JsonSerDe")+ .parameters(Map.of("serialization.format", "1"))+ .build())+ .columns(Column.builder()+ .name("id")+ .type("int")+ .build())+ .build())+ .build();+ }++ private static UserDefinedFunctionInput functionInput(String ownerName) {+ return UserDefinedFunctionInput.builder()+ .functionName(FUNCTION_NAME)+ .className("CatalogFunction")+ .ownerName(ownerName)+ .ownerType(PrincipalType.USER)+ .resourceUris(ResourceUri.builder()+ .resourceType(ResourceType.FILE)+ .uri("s3://floci-glue-catalog/function.jar")+ .build())+ .build();+ }+}