Light, fluffy, and always free - The AWS Local Emulator alternative
fix: archive Glue table versions (#1233)
David Phillips27943b35c347
diff --git a/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats b/compatibility-tests/sdk-test-awscli/test/glue-catalog.batsindex 367cb9d3c9..f415a92228 100644--- a/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats+++ b/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats@@ -168,6 +168,21 @@ function_input() {[ "$description" = "updated" ][ "$version_id" = "1" ]+ run aws_cmd glue get-table-versions \+ --database-name "$DB_NAME" \+ --table-name "$TABLE_NAME"+ assert_success+ version_count=$(echo "$output" | jq '.TableVersions | length')+ current_version=$(json_get "$output" '.TableVersions[0].VersionId')+ current_description=$(json_get "$output" '.TableVersions[0].Table.Description')+ archived_version=$(json_get "$output" '.TableVersions[1].VersionId')+ archived_description=$(json_get "$output" '.TableVersions[1].Table.Description')+ [ "$version_count" = "2" ]+ [ "$current_version" = "1" ]+ [ "$current_description" = "updated" ]+ [ "$archived_version" = "0" ]+ [ "$archived_description" = "created" ]+run aws_cmd glue create-partition \--database-name "$DB_NAME" \--table-name "$TABLE_NAME" \diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.javaindex c5ac29d831..a0a7eb145c 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java@@ -88,11 +88,14 @@ public class GlueJsonHandler {case "UpdateTable" -> {String dbName = request.get("DatabaseName").asText();Table table = mapper.treeToValue(request.get("TableInput"), Table.class);- glueService.updateTable(dbName, table, request.path("VersionId").asText(null));+ glueService.updateTable(dbName, table, request.path("VersionId").asText(null),+ request.path("SkipArchive").asBoolean(false));yield Response.ok().build();}case "GetTableVersions" -> {- yield Response.ok(Map.of("TableVersions", glueService.getTableVersions())).build();+ String dbName = request.get("DatabaseName").asText();+ String tableName = request.get("TableName").asText();+ yield Response.ok(Map.of("TableVersions", glueService.getTableVersions(dbName, tableName))).build();}case "DeleteTable" -> {String dbName = request.get("DatabaseName").asText();diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.javaindex 5b28158d85..25606564b5 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java@@ -41,6 +41,7 @@ public class GlueService {private final StorageBackend<String, Database> databaseStore;private final StorageBackend<String, Table> tableStore;+ private final StorageBackend<String, Table> tableVersionStore;private final StorageBackend<String, Partition> partitionStore;private final StorageBackend<String, UserDefinedFunction> functionStore;private final GlueSchemaRegistryService schemaRegistryService;@@ -52,6 +53,7 @@ public class GlueService {RegionResolver regionResolver) {this.databaseStore = storageFactory.create("glue", "databases.json", new TypeReference<>() {});this.tableStore = storageFactory.create("glue", "tables.json", new TypeReference<>() {});+ this.tableVersionStore = storageFactory.create("glue", "table_versions.json", new TypeReference<>() {});this.partitionStore = storageFactory.create("glue", "partitions.json", new TypeReference<>() {});this.functionStore = storageFactory.create("glue", "functions.json", new TypeReference<>() {});this.schemaRegistryService = schemaRegistryService;@@ -60,12 +62,14 @@ public class GlueService {GlueService(StorageBackend<String, Database> databaseStore,StorageBackend<String, Table> tableStore,+ StorageBackend<String, Table> tableVersionStore,StorageBackend<String, Partition> partitionStore,StorageBackend<String, UserDefinedFunction> functionStore,GlueSchemaRegistryService schemaRegistryService,RegionResolver regionResolver) {this.databaseStore = databaseStore;this.tableStore = tableStore;+ this.tableVersionStore = tableVersionStore;this.partitionStore = partitionStore;this.functionStore = functionStore;this.schemaRegistryService = schemaRegistryService;@@ -151,11 +155,7 @@ public class GlueService {return resolved;}- public void updateTable(String databaseName, Table table) {- updateTable(databaseName, table, null);- }-- public synchronized void updateTable(String databaseName, Table table, String versionId) {+ public synchronized void updateTable(String databaseName, Table table, String versionId, boolean skipArchive) {Database database = getDatabase(databaseName);String key = tableKey(databaseName, table.getName());Table existing = tableStore.get(key)@@ -165,6 +165,10 @@ public class GlueService {throw new AwsException("ConcurrentModificationException","Update table failed due to concurrent modifications.", 400);}+ if (!skipArchive) {+ tableVersionStore.put(tableVersionKey(existing.getDatabaseName(), existing.getName(), existing.getVersionId()),+ copyTable(existing));+ }validateSchemaReference(table);table.setName(normalizeName(table.getName()));table.setDatabaseName(database.getName());@@ -175,13 +179,27 @@ public class GlueService {LOG.infov("Updated Glue Table: {0}.{1}", databaseName, table.getName());}- public List<Map<String, Object>> getTableVersions() {- return List.of();+ public List<Map<String, Object>> getTableVersions(String databaseName, String tableName) {+ Table current = getTable(databaseName, tableName);+ String prefix = tableKey(databaseName, tableName) + ":";+ List<Table> versions = new ArrayList<>();+ versions.add(current);+ versions.addAll(tableVersionStore.scan(k -> k.startsWith(prefix)).stream()+ .sorted(Comparator.comparing(GlueService::versionIdAsLong).reversed())+ .toList());+ return versions.stream()+ .map(table -> Map.<String, Object>of(+ "Table", withResolvedSchemaReference(table),+ "VersionId", table.getVersionId()))+ .toList();}public void deleteTable(String databaseName, String tableName) {String key = tableKey(databaseName, tableName);tableStore.delete(key);+ tableVersionStore.keys().stream()+ .filter(versionKey -> versionKey.startsWith(key + ":"))+ .forEach(tableVersionStore::delete);partitionStore.scan(k -> k.startsWith(key + ":")).forEach(p -> {partitionStore.delete(key + ":" + String.join(",", p.getValues()));});@@ -336,6 +354,10 @@ public class GlueService {return normalizeName(databaseName) + ":" + normalizeName(tableName);}+ private static String tableVersionKey(String databaseName, String tableName, String versionId) {+ return tableKey(databaseName, tableName) + ":" + versionId;+ }+private static String normalizeName(String name) {return name.toLowerCase(Locale.ROOT);}@@ -412,6 +434,10 @@ public class GlueService {}}+ private static long versionIdAsLong(Table table) {+ return Long.parseLong(table.getVersionId());+ }+private static StorageDescriptor copyStorageDescriptor(StorageDescriptor source) {if (source == null) {return null;diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.javaindex 9c969145dc..99ca25df30 100644--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java@@ -46,6 +46,7 @@ class GlueServiceTest {private GlueService glueService;private GlueSchemaRegistryService schemaRegistryService;private StorageBackend<String, Table> tableStore;+ private StorageBackend<String, Table> tableVersionStore;private StorageBackend<String, Partition> partitionStore;@BeforeEach@@ -54,10 +55,12 @@ class GlueServiceTest {StorageFactory storageFactory = new InMemoryStorageFactory();schemaRegistryService = new GlueSchemaRegistryService(storageFactory, regionResolver);tableStore = new InMemoryStorage<>();+ tableVersionStore = new InMemoryStorage<>();partitionStore = new InMemoryStorage<>();glueService = new GlueService(new InMemoryStorage<String, Database>(),tableStore,+ tableVersionStore,partitionStore,new InMemoryStorage<String, UserDefinedFunction>(),schemaRegistryService, regionResolver);@@ -236,7 +239,7 @@ class GlueServiceTest {replacementSd.setColumns(java.util.List.of(new Column("b", "bigint")));replacement.setStorageDescriptor(replacementSd);- glueService.updateTable("db1", replacement);+ glueService.updateTable("db1", replacement, null, false);Table fetched = glueService.getTable("db1", "plain");assertEquals(created.getCreateTime(), fetched.getCreateTime());@@ -256,27 +259,27 @@ class GlueServiceTest {Table nonCanonicalVersionReplacement = new Table();nonCanonicalVersionReplacement.setName("plain");AwsException nonCanonicalVersionEx = assertThrows(AwsException.class,- () -> glueService.updateTable("db1", nonCanonicalVersionReplacement, "00"));+ () -> glueService.updateTable("db1", nonCanonicalVersionReplacement, "00", false));assertEquals("ConcurrentModificationException", nonCanonicalVersionEx.getErrorCode());assertEquals("0", glueService.getTable("db1", "plain").getVersionId());Table nonNumericVersionReplacement = new Table();nonNumericVersionReplacement.setName("plain");AwsException nonNumericVersionEx = assertThrows(AwsException.class,- () -> glueService.updateTable("db1", nonNumericVersionReplacement, "invalid"));+ () -> glueService.updateTable("db1", nonNumericVersionReplacement, "invalid", false));assertEquals("ConcurrentModificationException", nonNumericVersionEx.getErrorCode());assertEquals("0", glueService.getTable("db1", "plain").getVersionId());Table firstReplacement = new Table();firstReplacement.setName("plain");firstReplacement.setDescription("first");- glueService.updateTable("db1", firstReplacement, "0");+ glueService.updateTable("db1", firstReplacement, "0", false);assertEquals("1", glueService.getTable("db1", "plain").getVersionId());Table staleReplacement = new Table();staleReplacement.setName("plain");AwsException ex = assertThrows(AwsException.class,- () -> glueService.updateTable("db1", staleReplacement, "0"));+ () -> glueService.updateTable("db1", staleReplacement, "0", false));assertEquals("ConcurrentModificationException", ex.getErrorCode());assertEquals("Update table failed due to concurrent modifications.", ex.getMessage());@@ -291,7 +294,7 @@ class GlueServiceTest {Table replacement = new Table();replacement.setName("plain");- glueService.updateTable("db1", replacement);+ glueService.updateTable("db1", replacement, null, false);assertEquals("1", glueService.getTable("db1", "plain").getVersionId());}@@ -371,7 +374,7 @@ class GlueServiceTest {replacement.setName("plain");replacement.setParameters(Map.of("metadata_location", "s3://bucket/v2.metadata.json"));- glueService.updateTable("db1", replacement, created.getVersionId());+ glueService.updateTable("db1", replacement, created.getVersionId(), false);Table fetched = glueService.getTable("db1", "plain");assertEquals("1", fetched.getVersionId());@@ -390,7 +393,7 @@ class GlueServiceTest {replacement.setParameters(Map.of("metadata_location", "s3://bucket/v2.metadata.json"));AwsException ex = assertThrows(AwsException.class,- () -> glueService.updateTable("db1", replacement, "stale-version"));+ () -> glueService.updateTable("db1", replacement, "stale-version", false));assertEquals("ConcurrentModificationException", ex.getErrorCode());Table fetched = glueService.getTable("db1", "plain");@@ -420,13 +423,13 @@ class GlueServiceTest {Table firstReplacement = new Table();firstReplacement.setName("plain");firstReplacement.setParameters(Map.of("metadata_location", "s3://bucket/v2a.metadata.json"));- glueService.updateTable("db1", firstReplacement, versionId);+ glueService.updateTable("db1", firstReplacement, versionId, false);Table secondReplacement = new Table();secondReplacement.setName("plain");secondReplacement.setParameters(Map.of("metadata_location", "s3://bucket/v2b.metadata.json"));AwsException ex = assertThrows(AwsException.class,- () -> glueService.updateTable("db1", secondReplacement, versionId));+ () -> glueService.updateTable("db1", secondReplacement, versionId, false));assertEquals("ConcurrentModificationException", ex.getErrorCode());Table fetched = glueService.getTable("db1", "plain");@@ -466,7 +469,7 @@ class GlueServiceTest {Table replacement = new Table();replacement.setName("MIXEDCASETABLE");replacement.setDescription("updated");- glueService.updateTable("MIXEDCASEDATABASE", replacement, fetchedTable.getVersionId());+ glueService.updateTable("MIXEDCASEDATABASE", replacement, fetchedTable.getVersionId(), false);assertEquals("updated", glueService.getTable("mixedcasedatabase", "mixedcasetable").getDescription());Partition partition = new Partition();@@ -535,8 +538,40 @@ class GlueServiceTest {}@Test- void getTableVersionsReturnsEmptyListForAthenaCompatibility() {- assertTrue(glueService.getTableVersions().isEmpty());+ void getTableVersionsReturnsCurrentAndArchivedVersions() {+ Table table = new Table();+ table.setName("plain");+ table.setDescription("created");+ glueService.createTable("db1", table);++ Table replacement = new Table();+ replacement.setName("plain");+ replacement.setDescription("updated");+ glueService.updateTable("db1", replacement, "0", false);++ List<Map<String, Object>> tableVersions = glueService.getTableVersions("db1", "plain");++ assertEquals(2, tableVersions.size());+ assertEquals("1", tableVersions.get(0).get("VersionId"));+ assertEquals("updated", ((Table) tableVersions.get(0).get("Table")).getDescription());+ assertEquals("0", tableVersions.get(1).get("VersionId"));+ assertEquals("created", ((Table) tableVersions.get(1).get("Table")).getDescription());+ }++ @Test+ void updateTableSkipsArchiveWhenSkipArchiveIsTrue() {+ Table table = new Table();+ table.setName("plain");+ glueService.createTable("db1", table);++ Table replacement = new Table();+ replacement.setName("plain");+ glueService.updateTable("db1", replacement, "0", true);++ List<Map<String, Object>> tableVersions = glueService.getTableVersions("db1", "plain");++ assertEquals(1, tableVersions.size());+ assertEquals("1", tableVersions.get(0).get("VersionId"));}@Test