Light, fluffy, and always free - The AWS Local Emulator alternative
fix(athena): project the column names the Glue table declares (#3414)
Ovidiuebe9cf2264c6
diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilder.java b/src/main/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilder.javaindex 31fc24fdf3..0ac8693598 100644--- a/src/main/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilder.java+++ b/src/main/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilder.java@@ -1,14 +1,18 @@package io.github.hectorvent.floci.services.athena;import io.github.hectorvent.floci.services.glue.GlueService;+import io.github.hectorvent.floci.services.glue.model.Column;import io.github.hectorvent.floci.services.glue.model.Database;import io.github.hectorvent.floci.services.glue.model.Table;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.util.ArrayList;+import java.util.HashSet;import java.util.List;import java.util.Locale;+import java.util.Set;@ApplicationScopedpublic class GlueViewDdlBuilder {@@ -88,7 +92,9 @@ public class GlueViewDdlBuilder {: quote(t.getName());sb.append("CREATE OR REPLACE VIEW ").append(target)- .append(" AS SELECT * FROM ")+ .append(" AS SELECT ")+ .append(buildProjection(t))+ .append(" FROM ").append(readExpression(readFn, normalizedLocation)).append(";\n");} catch (Exception e) {@@ -97,6 +103,58 @@ public class GlueViewDdlBuilder {}}+ /**+ * Projects the columns the catalog declares, rather than whatever the underlying files spell.+ *+ * <p>Athena reports a table's declared column names: a JSON body written with {@code tenantId}+ * surfaces as {@code tenantid} when the table declares it that way, because the catalog owns the+ * schema and the SerDe maps onto it. Inference sees only the data, so without this the view+ * exposes the file's spelling and a client reading the declared name finds nothing. Identifier+ * resolution is case-insensitive here, so a declared name still binds to an inferred one that+ * differs from it only by case.+ *+ * <p>Partition keys are part of the table's schema and are what partition predicates filter on,+ * so they are projected too. A table that declares no columns keeps {@code *}, leaving inference+ * in charge.+ */+ static String buildProjection(Table table) {+ List<Column> declared = new ArrayList<>();+ if (table != null) {+ if (table.getStorageDescriptor() != null && table.getStorageDescriptor().getColumns() != null) {+ declared.addAll(table.getStorageDescriptor().getColumns());+ }+ if (table.getPartitionKeys() != null) {+ declared.addAll(table.getPartitionKeys());+ }+ }++ List<String> names = new ArrayList<>();+ Set<String> seen = new HashSet<>();+ for (Column c : declared) {+ if (c == null || c.getName() == null || c.getName().isBlank()) {+ continue;+ }+ String name = c.getName();+ // A partition key repeating a data column would bind twice and make the view ambiguous.+ if (seen.add(name.toLowerCase(Locale.ROOT))) {+ names.add(name);+ }+ }++ if (names.isEmpty()) {+ return "*";+ }++ StringBuilder sb = new StringBuilder();+ for (int i = 0; i < names.size(); i++) {+ if (i > 0) {+ sb.append(", ");+ }+ sb.append(quote(names.get(i))).append(" AS ").append(quote(names.get(i)));+ }+ return sb.toString();+ }+static String inferReadFunction(Table table) {if (table == null || table.getStorageDescriptor() == null) {return "read_csv_auto";diff --git a/src/test/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilderTest.java b/src/test/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilderTest.javaindex 4e65ad153e..e9978de508 100644--- a/src/test/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilderTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilderTest.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.athena;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.glue.GlueService;+import io.github.hectorvent.floci.services.glue.model.Column;import io.github.hectorvent.floci.services.glue.model.Database;import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;import io.github.hectorvent.floci.services.glue.model.Table;@@ -50,6 +51,82 @@ class GlueViewDdlBuilderTest {return table;}+ private static Column column(String name) {+ Column c = new Column();+ c.setName(name);+ c.setType("string");+ return c;+ }++ private Table tableWithColumns(String name, String location, List<Column> columns, List<Column> partitionKeys) {+ Table table = createTable(name, location, "org.openx.data.jsonserde.JsonSerDe", null);+ table.getStorageDescriptor().setColumns(columns);+ table.setPartitionKeys(partitionKeys);+ return table;+ }++ private String ddlFor(Table table) {+ Database db = createDatabase("audit");+ when(glueService.getDatabases()).thenReturn(List.of(db));+ when(glueService.getTables("audit")).thenReturn(List.of(table));+ return builder.build(null);+ }++ /**+ * Athena reports the column names the catalog declares. The files here spell them camelCase, so a+ * view built by inference alone would expose the file's spelling and a client reading the declared+ * name would find nothing.+ */+ @Test+ void testDeclaredColumnsAreProjectedInsteadOfStar() {+ String ddl = ddlFor(tableWithColumns("audit_events", "s3://audit/",+ List.of(column("eventid"), column("tenantid")),+ List.of(column("tenant"), column("year"))));++ assertTrue(ddl.contains("\"eventid\" AS \"eventid\""), ddl);+ assertTrue(ddl.contains("\"tenantid\" AS \"tenantid\""), ddl);+ assertFalse(ddl.contains("SELECT * FROM"), ddl);+ }++ /** Partition keys are part of the schema and are what partition predicates filter on. */+ @Test+ void testPartitionKeysAreProjectedAlongsideDataColumns() {+ String ddl = ddlFor(tableWithColumns("audit_events", "s3://audit/",+ List.of(column("eventid")),+ List.of(column("tenant"), column("year"), column("month"), column("day"))));++ for (String name : List.of("eventid", "tenant", "year", "month", "day")) {+ assertTrue(ddl.contains("\"" + name + "\" AS \"" + name + "\""), name + " missing from: " + ddl);+ }+ }++ /** A partition key repeating a data column would bind twice and make the view ambiguous. */+ @Test+ void testDuplicateColumnAndPartitionKeyIsProjectedOnce() {+ String ddl = ddlFor(tableWithColumns("t", "s3://b/",+ List.of(column("tenant")), List.of(column("TENANT"))));++ int first = ddl.indexOf("AS \"tenant\"");+ assertTrue(first >= 0, ddl);+ assertEquals(-1, ddl.indexOf("AS \"tenant\"", first + 1), "projected twice: " + ddl);+ }++ /** A table that declares no columns leaves inference in charge, as before. */+ @Test+ void testTableWithoutDeclaredColumnsStillUsesStar() {+ String ddl = ddlFor(tableWithColumns("t", "s3://b/", null, null));++ assertTrue(ddl.contains("SELECT * FROM"), ddl);+ }++ /** Column identifiers go through the same quoting as table and schema names. */+ @Test+ void testColumnIdentifierQuotingEscapesDoubleQuotes() {+ String ddl = ddlFor(tableWithColumns("t", "s3://b/", List.of(column("we\"ird")), null));++ assertTrue(ddl.contains("\"we\"\"ird\" AS \"we\"\"ird\""), ddl);+ }+@Testvoid testTwoDatabasesQualifiedViewsAndSchemaPerDb() {Database shopDb = createDatabase("shop");