Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix: archive Glue table versions (#1233)

David Phillips27943b35c347
diff --git a/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats b/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats
index 367cb9d3c9..f415a92228 100644
--- a/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats
+++ b/compatibility-tests/sdk-test-awscli/test/glue-catalog.bats
@@ -168,6 +168,21 @@ function_input() {
[ "$description" = "updated" ]
[ "$version_id" = "1" ]
+ run aws_cmd glue get-table-versions \
+ --database-name "$DB_NAME" \
+ --table-name "$TABLE_NAME"
+ assert_success
+ version_count=$(echo "$output" | jq '.TableVersions | length')
+ current_version=$(json_get "$output" '.TableVersions[0].VersionId')
+ current_description=$(json_get "$output" '.TableVersions[0].Table.Description')
+ archived_version=$(json_get "$output" '.TableVersions[1].VersionId')
+ archived_description=$(json_get "$output" '.TableVersions[1].Table.Description')
+ [ "$version_count" = "2" ]
+ [ "$current_version" = "1" ]
+ [ "$current_description" = "updated" ]
+ [ "$archived_version" = "0" ]
+ [ "$archived_description" = "created" ]
+
run aws_cmd glue create-partition \
--database-name "$DB_NAME" \
--table-name "$TABLE_NAME" \
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
index c5ac29d831..a0a7eb145c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueJsonHandler.java
@@ -88,11 +88,14 @@ public class GlueJsonHandler {
case "UpdateTable" -> {
String dbName = request.get("DatabaseName").asText();
Table table = mapper.treeToValue(request.get("TableInput"), Table.class);
- glueService.updateTable(dbName, table, request.path("VersionId").asText(null));
+ glueService.updateTable(dbName, table, request.path("VersionId").asText(null),
+ request.path("SkipArchive").asBoolean(false));
yield Response.ok().build();
}
case "GetTableVersions" -> {
- yield Response.ok(Map.of("TableVersions", glueService.getTableVersions())).build();
+ String dbName = request.get("DatabaseName").asText();
+ String tableName = request.get("TableName").asText();
+ yield Response.ok(Map.of("TableVersions", glueService.getTableVersions(dbName, tableName))).build();
}
case "DeleteTable" -> {
String dbName = request.get("DatabaseName").asText();
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
index 5b28158d85..25606564b5 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java
@@ -41,6 +41,7 @@ public class GlueService {
private final StorageBackend<String, Database> databaseStore;
private final StorageBackend<String, Table> tableStore;
+ private final StorageBackend<String, Table> tableVersionStore;
private final StorageBackend<String, Partition> partitionStore;
private final StorageBackend<String, UserDefinedFunction> functionStore;
private final GlueSchemaRegistryService schemaRegistryService;
@@ -52,6 +53,7 @@ public class GlueService {
RegionResolver regionResolver) {
this.databaseStore = storageFactory.create("glue", "databases.json", new TypeReference<>() {});
this.tableStore = storageFactory.create("glue", "tables.json", new TypeReference<>() {});
+ this.tableVersionStore = storageFactory.create("glue", "table_versions.json", new TypeReference<>() {});
this.partitionStore = storageFactory.create("glue", "partitions.json", new TypeReference<>() {});
this.functionStore = storageFactory.create("glue", "functions.json", new TypeReference<>() {});
this.schemaRegistryService = schemaRegistryService;
@@ -60,12 +62,14 @@ public class GlueService {
GlueService(StorageBackend<String, Database> databaseStore,
StorageBackend<String, Table> tableStore,
+ StorageBackend<String, Table> tableVersionStore,
StorageBackend<String, Partition> partitionStore,
StorageBackend<String, UserDefinedFunction> functionStore,
GlueSchemaRegistryService schemaRegistryService,
RegionResolver regionResolver) {
this.databaseStore = databaseStore;
this.tableStore = tableStore;
+ this.tableVersionStore = tableVersionStore;
this.partitionStore = partitionStore;
this.functionStore = functionStore;
this.schemaRegistryService = schemaRegistryService;
@@ -151,11 +155,7 @@ public class GlueService {
return resolved;
}
- public void updateTable(String databaseName, Table table) {
- updateTable(databaseName, table, null);
- }
-
- public synchronized void updateTable(String databaseName, Table table, String versionId) {
+ public synchronized void updateTable(String databaseName, Table table, String versionId, boolean skipArchive) {
Database database = getDatabase(databaseName);
String key = tableKey(databaseName, table.getName());
Table existing = tableStore.get(key)
@@ -165,6 +165,10 @@ public class GlueService {
throw new AwsException("ConcurrentModificationException",
"Update table failed due to concurrent modifications.", 400);
}
+ if (!skipArchive) {
+ tableVersionStore.put(tableVersionKey(existing.getDatabaseName(), existing.getName(), existing.getVersionId()),
+ copyTable(existing));
+ }
validateSchemaReference(table);
table.setName(normalizeName(table.getName()));
table.setDatabaseName(database.getName());
@@ -175,13 +179,27 @@ public class GlueService {
LOG.infov("Updated Glue Table: {0}.{1}", databaseName, table.getName());
}
- public List<Map<String, Object>> getTableVersions() {
- return List.of();
+ public List<Map<String, Object>> getTableVersions(String databaseName, String tableName) {
+ Table current = getTable(databaseName, tableName);
+ String prefix = tableKey(databaseName, tableName) + ":";
+ List<Table> versions = new ArrayList<>();
+ versions.add(current);
+ versions.addAll(tableVersionStore.scan(k -> k.startsWith(prefix)).stream()
+ .sorted(Comparator.comparing(GlueService::versionIdAsLong).reversed())
+ .toList());
+ return versions.stream()
+ .map(table -> Map.<String, Object>of(
+ "Table", withResolvedSchemaReference(table),
+ "VersionId", table.getVersionId()))
+ .toList();
}
public void deleteTable(String databaseName, String tableName) {
String key = tableKey(databaseName, tableName);
tableStore.delete(key);
+ tableVersionStore.keys().stream()
+ .filter(versionKey -> versionKey.startsWith(key + ":"))
+ .forEach(tableVersionStore::delete);
partitionStore.scan(k -> k.startsWith(key + ":")).forEach(p -> {
partitionStore.delete(key + ":" + String.join(",", p.getValues()));
});
@@ -336,6 +354,10 @@ public class GlueService {
return normalizeName(databaseName) + ":" + normalizeName(tableName);
}
+ private static String tableVersionKey(String databaseName, String tableName, String versionId) {
+ return tableKey(databaseName, tableName) + ":" + versionId;
+ }
+
private static String normalizeName(String name) {
return name.toLowerCase(Locale.ROOT);
}
@@ -412,6 +434,10 @@ public class GlueService {
}
}
+ private static long versionIdAsLong(Table table) {
+ return Long.parseLong(table.getVersionId());
+ }
+
private static StorageDescriptor copyStorageDescriptor(StorageDescriptor source) {
if (source == null) {
return null;
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
index 9c969145dc..99ca25df30 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/GlueServiceTest.java
@@ -46,6 +46,7 @@ class GlueServiceTest {
private GlueService glueService;
private GlueSchemaRegistryService schemaRegistryService;
private StorageBackend<String, Table> tableStore;
+ private StorageBackend<String, Table> tableVersionStore;
private StorageBackend<String, Partition> partitionStore;
@BeforeEach
@@ -54,10 +55,12 @@ class GlueServiceTest {
StorageFactory storageFactory = new InMemoryStorageFactory();
schemaRegistryService = new GlueSchemaRegistryService(storageFactory, regionResolver);
tableStore = new InMemoryStorage<>();
+ tableVersionStore = new InMemoryStorage<>();
partitionStore = new InMemoryStorage<>();
glueService = new GlueService(
new InMemoryStorage<String, Database>(),
tableStore,
+ tableVersionStore,
partitionStore,
new InMemoryStorage<String, UserDefinedFunction>(),
schemaRegistryService, regionResolver);
@@ -236,7 +239,7 @@ class GlueServiceTest {
replacementSd.setColumns(java.util.List.of(new Column("b", "bigint")));
replacement.setStorageDescriptor(replacementSd);
- glueService.updateTable("db1", replacement);
+ glueService.updateTable("db1", replacement, null, false);
Table fetched = glueService.getTable("db1", "plain");
assertEquals(created.getCreateTime(), fetched.getCreateTime());
@@ -256,27 +259,27 @@ class GlueServiceTest {
Table nonCanonicalVersionReplacement = new Table();
nonCanonicalVersionReplacement.setName("plain");
AwsException nonCanonicalVersionEx = assertThrows(AwsException.class,
- () -> glueService.updateTable("db1", nonCanonicalVersionReplacement, "00"));
+ () -> glueService.updateTable("db1", nonCanonicalVersionReplacement, "00", false));
assertEquals("ConcurrentModificationException", nonCanonicalVersionEx.getErrorCode());
assertEquals("0", glueService.getTable("db1", "plain").getVersionId());
Table nonNumericVersionReplacement = new Table();
nonNumericVersionReplacement.setName("plain");
AwsException nonNumericVersionEx = assertThrows(AwsException.class,
- () -> glueService.updateTable("db1", nonNumericVersionReplacement, "invalid"));
+ () -> glueService.updateTable("db1", nonNumericVersionReplacement, "invalid", false));
assertEquals("ConcurrentModificationException", nonNumericVersionEx.getErrorCode());
assertEquals("0", glueService.getTable("db1", "plain").getVersionId());
Table firstReplacement = new Table();
firstReplacement.setName("plain");
firstReplacement.setDescription("first");
- glueService.updateTable("db1", firstReplacement, "0");
+ glueService.updateTable("db1", firstReplacement, "0", false);
assertEquals("1", glueService.getTable("db1", "plain").getVersionId());
Table staleReplacement = new Table();
staleReplacement.setName("plain");
AwsException ex = assertThrows(AwsException.class,
- () -> glueService.updateTable("db1", staleReplacement, "0"));
+ () -> glueService.updateTable("db1", staleReplacement, "0", false));
assertEquals("ConcurrentModificationException", ex.getErrorCode());
assertEquals("Update table failed due to concurrent modifications.", ex.getMessage());
@@ -291,7 +294,7 @@ class GlueServiceTest {
Table replacement = new Table();
replacement.setName("plain");
- glueService.updateTable("db1", replacement);
+ glueService.updateTable("db1", replacement, null, false);
assertEquals("1", glueService.getTable("db1", "plain").getVersionId());
}
@@ -371,7 +374,7 @@ class GlueServiceTest {
replacement.setName("plain");
replacement.setParameters(Map.of("metadata_location", "s3://bucket/v2.metadata.json"));
- glueService.updateTable("db1", replacement, created.getVersionId());
+ glueService.updateTable("db1", replacement, created.getVersionId(), false);
Table fetched = glueService.getTable("db1", "plain");
assertEquals("1", fetched.getVersionId());
@@ -390,7 +393,7 @@ class GlueServiceTest {
replacement.setParameters(Map.of("metadata_location", "s3://bucket/v2.metadata.json"));
AwsException ex = assertThrows(AwsException.class,
- () -> glueService.updateTable("db1", replacement, "stale-version"));
+ () -> glueService.updateTable("db1", replacement, "stale-version", false));
assertEquals("ConcurrentModificationException", ex.getErrorCode());
Table fetched = glueService.getTable("db1", "plain");
@@ -420,13 +423,13 @@ class GlueServiceTest {
Table firstReplacement = new Table();
firstReplacement.setName("plain");
firstReplacement.setParameters(Map.of("metadata_location", "s3://bucket/v2a.metadata.json"));
- glueService.updateTable("db1", firstReplacement, versionId);
+ glueService.updateTable("db1", firstReplacement, versionId, false);
Table secondReplacement = new Table();
secondReplacement.setName("plain");
secondReplacement.setParameters(Map.of("metadata_location", "s3://bucket/v2b.metadata.json"));
AwsException ex = assertThrows(AwsException.class,
- () -> glueService.updateTable("db1", secondReplacement, versionId));
+ () -> glueService.updateTable("db1", secondReplacement, versionId, false));
assertEquals("ConcurrentModificationException", ex.getErrorCode());
Table fetched = glueService.getTable("db1", "plain");
@@ -466,7 +469,7 @@ class GlueServiceTest {
Table replacement = new Table();
replacement.setName("MIXEDCASETABLE");
replacement.setDescription("updated");
- glueService.updateTable("MIXEDCASEDATABASE", replacement, fetchedTable.getVersionId());
+ glueService.updateTable("MIXEDCASEDATABASE", replacement, fetchedTable.getVersionId(), false);
assertEquals("updated", glueService.getTable("mixedcasedatabase", "mixedcasetable").getDescription());
Partition partition = new Partition();
@@ -535,8 +538,40 @@ class GlueServiceTest {
}
@Test
- void getTableVersionsReturnsEmptyListForAthenaCompatibility() {
- assertTrue(glueService.getTableVersions().isEmpty());
+ void getTableVersionsReturnsCurrentAndArchivedVersions() {
+ Table table = new Table();
+ table.setName("plain");
+ table.setDescription("created");
+ glueService.createTable("db1", table);
+
+ Table replacement = new Table();
+ replacement.setName("plain");
+ replacement.setDescription("updated");
+ glueService.updateTable("db1", replacement, "0", false);
+
+ List<Map<String, Object>> tableVersions = glueService.getTableVersions("db1", "plain");
+
+ assertEquals(2, tableVersions.size());
+ assertEquals("1", tableVersions.get(0).get("VersionId"));
+ assertEquals("updated", ((Table) tableVersions.get(0).get("Table")).getDescription());
+ assertEquals("0", tableVersions.get(1).get("VersionId"));
+ assertEquals("created", ((Table) tableVersions.get(1).get("Table")).getDescription());
+ }
+
+ @Test
+ void updateTableSkipsArchiveWhenSkipArchiveIsTrue() {
+ Table table = new Table();
+ table.setName("plain");
+ glueService.createTable("db1", table);
+
+ Table replacement = new Table();
+ replacement.setName("plain");
+ glueService.updateTable("db1", replacement, "0", true);
+
+ List<Map<String, Object>> tableVersions = glueService.getTableVersions("db1", "plain");
+
+ assertEquals(1, tableVersions.size());
+ assertEquals("1", tableVersions.get(0).get("VersionId"));
}
@Test