Light, fluffy, and always free - The AWS Local Emulator alternative
fix(athena): read a projecting table's partition paths, not its whole location (#3415)
Ovidiu6d6940bb8fc3
diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.javaindex 6517470515..1f26cf11d7 100644--- a/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java+++ b/src/main/java/io/github/hectorvent/floci/services/athena/AthenaService.java@@ -146,6 +146,9 @@ public class AthenaService {// Submit async — caller gets the ID immediately while execution runs in backgroundvertx.executeBlocking(() -> {+ // Athena rejects a query that leaves an injected partition column unconstrained. It fails+ // the query rather than the submission, which is what throwing here produces.+ PartitionProjection.assertInjectedColumnsFiltered(query, tablesForProjectionCheck(database));String setupDdl = ddlBuilder.build(database);if (outputLocation != null) {ensureOutputBucket(outputLocation);@@ -163,6 +166,20 @@ public class AthenaService {return id;}+ /** Tables of the query's database, or none when the catalog cannot answer. */+ private List<Table> tablesForProjectionCheck(String database) {+ if (database == null || database.isBlank()) {+ return List.of();+ }+ try {+ List<Table> tables = glueService.getTables(database);+ return tables == null ? List.of() : tables;+ } catch (Exception e) {+ LOG.debugv("Could not fetch tables for projection check on {0}: {1}", database, e.getMessage());+ return List.of();+ }+ }+public QueryExecution getQueryExecution(String id) {return queryStore.get(id).orElseThrow(() -> new AwsException("InvalidRequestException",diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilder.java b/src/main/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilder.javaindex 0ac8693598..0cf84cbeb0 100644--- a/src/main/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilder.java+++ b/src/main/java/io/github/hectorvent/floci/services/athena/GlueViewDdlBuilder.java@@ -87,6 +87,7 @@ public class GlueViewDdlBuilder {? location.substring(0, location.length() - 1): location;String readFn = inferReadFunction(t);+ String readPath = PartitionProjection.readPath(t, normalizedLocation);String target = qualified? quote(schemaOrNull) + "." + quote(t.getName()): quote(t.getName());@@ -95,7 +96,7 @@ public class GlueViewDdlBuilder {.append(" AS SELECT ").append(buildProjection(t)).append(" FROM ")- .append(readExpression(readFn, normalizedLocation))+ .append(readExpression(readFn, readPath)).append(";\n");} catch (Exception e) {LOG.debugv("skip Glue table {0}.{1}: {2}", schemaOrNull, t != null ? t.getName() : "unknown", e.getMessage());diff --git a/src/main/java/io/github/hectorvent/floci/services/athena/PartitionProjection.java b/src/main/java/io/github/hectorvent/floci/services/athena/PartitionProjection.javanew file mode 100644index 0000000000..bb3d61d211--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/athena/PartitionProjection.java@@ -0,0 +1,258 @@+package io.github.hectorvent.floci.services.athena;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.glue.model.Column;+import io.github.hectorvent.floci.services.glue.model.Table;++import java.util.ArrayList;+import java.util.HashSet;+import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.regex.Matcher;+import java.util.regex.Pattern;++/**+ * Partition projection: the table properties describing where a table's partitions live and what+ * values they take, so the partitions need not be listed from the catalog or from storage.+ */+final class PartitionProjection {++ /** Table property switching partition projection on. */+ private static final String ENABLED = "projection.enabled";++ /** Table property giving the partition path layout when it is not the default. */+ private static final String LOCATION_TEMPLATE = "storage.location.template";++ /** {@code ${column}} placeholders in a partition location template. */+ private static final Pattern TEMPLATE_PLACEHOLDER = Pattern.compile("\\$\\{[^}]*}");++ /**+ * Not part of a longer identifier, and not a quoted one either: {@code "limit"} is a column+ * named after a keyword, and a clause keyword is never written in quotes.+ */+ private static final String KEYWORD_BOUNDS = "(?<![A-Za-z0-9_\"`])%s(?![A-Za-z0-9_\"`])";++ /** The {@code WHERE} keyword, as a keyword rather than as an identifier spelling it. */+ private static final Pattern WHERE_KEYWORD = Pattern.compile(+ KEYWORD_BOUNDS.formatted("WHERE"), Pattern.CASE_INSENSITIVE);++ /** The keywords that can end a {@code WHERE} clause by starting the next one. */+ private static final Pattern CLAUSE_AFTER_WHERE = Pattern.compile(+ KEYWORD_BOUNDS.formatted(+ "(?:GROUP|HAVING|ORDER|WINDOW|LIMIT|OFFSET|FETCH|UNION|INTERSECT|EXCEPT)"),+ Pattern.CASE_INSENSITIVE);++ private PartitionProjection() {+ }++ static boolean enabled(Table table) {+ return table != null && "true".equalsIgnoreCase(parameter(table, ENABLED));+ }++ /** Case-insensitive lookup, since table properties are not normalised on the way in. */+ static String parameter(Table table, String name) {+ Map<String, String> params = table == null ? null : table.getParameters();+ if (params == null) {+ return null;+ }+ for (Map.Entry<String, String> e : params.entrySet()) {+ if (e.getKey() != null && e.getKey().equalsIgnoreCase(name)) {+ return e.getValue();+ }+ }+ return null;+ }++ /**+ * Returns the path to read for {@code table}: its partition layout when the table projects its+ * partitions, and {@code normalizedLocation} unchanged otherwise.+ *+ * <p>A projecting table's data lives only under its partition paths, and those are what Athena+ * reads - it builds them from the projection configuration rather than listing the location.+ * Reading the location wholesale instead picks up whatever else is stored beside the data, which+ * for a table rooted at a bucket means that bucket's other prefixes. A workgroup writing query+ * results under the same root is the common way this bites: the first query succeeds, its output+ * lands inside the read path, and the next query fails trying to read it as table data.+ *+ * <p>The layout comes from {@code storage.location.template} when set, and otherwise from the+ * default {@code column=value} form over the partition keys. Values stay wildcards: narrowing+ * them to the partitions a query actually needs requires that query's predicates, so the read+ * stays wide and the query itself does the filtering.+ */+ static String readPath(Table table, String normalizedLocation) {+ if (!enabled(table)) {+ return normalizedLocation;+ }++ String template = parameter(table, LOCATION_TEMPLATE);+ if (template != null && !template.isBlank()) {+ String wildcarded = TEMPLATE_PLACEHOLDER.matcher(template).replaceAll("*");+ return stripTrailingSlash(wildcarded);+ }++ List<Column> keys = table.getPartitionKeys();+ if (keys == null || keys.isEmpty()) {+ return normalizedLocation;+ }+ StringBuilder sb = new StringBuilder(normalizedLocation);+ for (Column key : keys) {+ if (key == null || key.getName() == null || key.getName().isBlank()) {+ return normalizedLocation;+ }+ sb.append('/').append(key.getName()).append("=*");+ }+ return sb.toString();+ }++ /**+ * Fails a query that names a projecting table but never constrains one of its {@code injected}+ * partition columns in a {@code WHERE} clause.+ *+ * <p>An injected column has no generatable range: its values come from the query, so Athena+ * requires a static equality condition on it in the {@code WHERE} clause and rejects the query+ * otherwise. Deciding whether a condition is a <em>static equality</em> one needs the query's+ * predicate tree, which is not available here, so this checks only whether the column is+ * mentioned inside a {@code WHERE} clause - any of them, including a subquery's. That is+ * strictly narrower than the real rule - a query filtering an injected column with, say, a+ * range condition is rejected by Athena and accepted here - but it never rejects a query Athena+ * would accept, and it catches the case that silently reads every partition.+ *+ * <p>Scoping to the {@code WHERE} clause is what makes the check worth having. A mention+ * anywhere in the query text also matches the column's own appearance in a {@code SELECT} list+ * or a {@code GROUP BY}, so {@code SELECT tenant, count(*) FROM audit_events GROUP BY tenant}+ * would pass while filtering nothing at all - exactly the case this exists to catch.+ */+ static void assertInjectedColumnsFiltered(String query, List<Table> tables) {+ if (query == null || tables == null) {+ return;+ }+ // Literals and comments are not code: a column name inside either constrains nothing.+ String code = maskLiteralsAndComments(query);+ List<String> whereClauses = null;+ for (Table table : tables) {+ if (!enabled(table) || table.getName() == null || table.getPartitionKeys() == null) {+ continue;+ }+ // A table the query never names cannot be constrained by it, and must not fail it.+ if (!mentions(code, table.getName())) {+ continue;+ }+ for (Column key : table.getPartitionKeys()) {+ if (key == null || key.getName() == null) {+ continue;+ }+ String type = parameter(table, "projection." + key.getName() + ".type");+ if (!"injected".equalsIgnoreCase(type)) {+ continue;+ }+ if (whereClauses == null) {+ whereClauses = whereClauses(code);+ }+ if (whereClauses.stream().anyMatch(clause -> mentions(clause, key.getName()))) {+ continue;+ }+ throw new AwsException("InvalidRequestException",+ "CONSTRAINT_VIOLATION: For the injected projected partition column "+ + key.getName() + ", the WHERE clause must contain only static equality "+ + "conditions, and at least one such condition must be present.", 400);+ }+ }+ }++ /**+ * The text of every {@code WHERE} clause in {@code code}, a subquery's included.+ *+ * <p>A clause runs from the keyword to whatever ends it: a following clause keyword, the+ * {@code )} closing the subquery that holds it, a statement terminator, or the end of the+ * query. Parentheses opened inside the clause are part of it, so a predicate with its own+ * subquery is not cut short.+ */+ private static List<String> whereClauses(String code) {+ Set<Integer> clauseStarts = new HashSet<>();+ Matcher boundaries = CLAUSE_AFTER_WHERE.matcher(code);+ while (boundaries.find()) {+ clauseStarts.add(boundaries.start());+ }++ List<String> clauses = new ArrayList<>();+ Matcher where = WHERE_KEYWORD.matcher(code);+ while (where.find()) {+ int start = where.end();+ int depth = 0;+ int i = start;+ while (i < code.length()) {+ char c = code.charAt(i);+ if (c == '(') {+ depth++;+ } else if (c == ')') {+ if (depth == 0) {+ break;+ }+ depth--;+ } else if (depth == 0 && (c == ';' || clauseStarts.contains(i))) {+ break;+ }+ i++;+ }+ clauses.add(code.substring(start, i));+ }+ return clauses;+ }++ /**+ * Blanks out single-quoted literals and comments, leaving everything else - and every index -+ * where it was. Double-quoted and backquoted identifiers stay: those are references to a column,+ * not text that happens to spell its name.+ */+ private static String maskLiteralsAndComments(String sql) {+ char[] chars = sql.toCharArray();+ int i = 0;+ while (i < chars.length) {+ int end;+ if (chars[i] == '\'') {+ end = i + 1;+ while (end < chars.length) {+ if (chars[end] != '\'') {+ end++;+ } else if (end + 1 < chars.length && chars[end + 1] == '\'') {+ end += 2; // an escaped quote, still inside the literal+ } else {+ end++;+ break;+ }+ }+ } else if (chars[i] == '-' && i + 1 < chars.length && chars[i + 1] == '-') {+ end = i;+ while (end < chars.length && chars[end] != '\n') {+ end++;+ }+ } else if (chars[i] == '/' && i + 1 < chars.length && chars[i + 1] == '*') {+ end = i + 2;+ while (end + 1 < chars.length && !(chars[end] == '*' && chars[end + 1] == '/')) {+ end++;+ }+ end = Math.min(chars.length, end + 2);+ } else {+ i++;+ continue;+ }+ for (int k = i; k < end; k++) {+ if (chars[k] != '\n') {+ chars[k] = ' ';+ }+ }+ i = end;+ }+ return new String(chars);+ }++ private static boolean mentions(String code, String identifier) {+ return Pattern.compile("(?<![A-Za-z0-9_])" + Pattern.quote(identifier) + "(?![A-Za-z0-9_])",+ Pattern.CASE_INSENSITIVE).matcher(code).find();+ }++ private static String stripTrailingSlash(String value) {+ return value.endsWith("/") ? value.substring(0, value.length() - 1) : value;+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/athena/PartitionProjectionTest.java b/src/test/java/io/github/hectorvent/floci/services/athena/PartitionProjectionTest.javanew file mode 100644index 0000000000..ce8815a129--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/athena/PartitionProjectionTest.java@@ -0,0 +1,210 @@+package io.github.hectorvent.floci.services.athena;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.glue.model.Column;+import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;+import io.github.hectorvent.floci.services.glue.model.Table;+import org.junit.jupiter.api.Test;++import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;++class PartitionProjectionTest {++ private static final String BUCKET = "s3://audit-logs";++ private static Column column(String name) {+ Column c = new Column();+ c.setName(name);+ c.setType("string");+ return c;+ }++ private static Table table(String name, Map<String, String> params, List<Column> partitionKeys) {+ Table t = new Table();+ t.setName(name);+ StorageDescriptor sd = new StorageDescriptor();+ sd.setLocation(BUCKET + "/");+ t.setStorageDescriptor(sd);+ t.setParameters(params);+ t.setPartitionKeys(partitionKeys);+ return t;+ }++ private static Map<String, String> projecting(String... extra) {+ Map<String, String> m = new LinkedHashMap<>();+ m.put("projection.enabled", "true");+ for (int i = 0; i + 1 < extra.length; i += 2) {+ m.put(extra[i], extra[i + 1]);+ }+ return m;+ }++ // ---- read path ----++ /**+ * A workgroup writing results under the table's root is the case that breaks: reading the root+ * wholesale picks the results back up as though they were table data.+ */+ @Test+ void templateGivesAPartitionPathThatExcludesSiblingPrefixes() {+ Table t = table("audit_events", projecting(+ "storage.location.template", BUCKET + "/tenant=${tenant}/year=${year}/month=${month}/day=${day}"),+ List.of(column("tenant"), column("year"), column("month"), column("day")));++ String path = PartitionProjection.readPath(t, BUCKET);++ assertEquals(BUCKET + "/tenant=*/year=*/month=*/day=*", path);+ assertTrue(path.startsWith(BUCKET + "/tenant="), path);+ }++ /** Without a template the default column=value layout is built from the partition keys. */+ @Test+ void partitionKeysGiveTheDefaultLayoutWhenNoTemplateIsSet() {+ Table t = table("t", projecting(), List.of(column("tenant"), column("year")));++ assertEquals(BUCKET + "/tenant=*/year=*", PartitionProjection.readPath(t, BUCKET));+ }++ /** Projection off means the location is read as before. */+ @Test+ void withoutProjectionTheLocationIsUnchanged() {+ Table t = table("t", Map.of(), List.of(column("tenant")));++ assertEquals(BUCKET, PartitionProjection.readPath(t, BUCKET));+ }++ /** A projecting table declaring no partitions has no layout to build. */+ @Test+ void projectionWithoutPartitionKeysLeavesTheLocationUnchanged() {+ assertEquals(BUCKET, PartitionProjection.readPath(table("t", projecting(), null), BUCKET));+ }++ @Test+ void trailingSlashOnTheTemplateIsDropped() {+ Table t = table("t", projecting("storage.location.template", BUCKET + "/tenant=${tenant}/"),+ List.of(column("tenant")));++ assertEquals(BUCKET + "/tenant=*", PartitionProjection.readPath(t, BUCKET));+ }++ // ---- injected columns ----++ private Table injectedTenantTable() {+ return table("audit_events", projecting("projection.tenant.type", "injected"),+ List.of(column("tenant")));+ }++ @Test+ void queryWithoutAnyMentionOfAnInjectedColumnIsRejected() {+ AwsException e = assertThrows(AwsException.class, () ->+ PartitionProjection.assertInjectedColumnsFiltered(+ "SELECT * FROM audit_events", List.of(injectedTenantTable())));++ assertTrue(e.getMessage().startsWith("CONSTRAINT_VIOLATION:"), e.getMessage());+ assertTrue(e.getMessage().contains("tenant"), e.getMessage());+ }++ @Test+ void queryConstrainingTheInjectedColumnIsAccepted() {+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(+ "SELECT * FROM audit_events WHERE tenant = 'abc'", List.of(injectedTenantTable())));+ }++ /** A table the query never names must not fail it. */+ @Test+ void unrelatedQueryAgainstAnotherTableIsUnaffected() {+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(+ "SELECT * FROM other_table", List.of(injectedTenantTable())));+ }++ /** A non-injected projected column generates its own values and needs no filter. */+ @Test+ void nonInjectedPartitionColumnsNeedNoFilter() {+ Table t = table("t", projecting("projection.year.type", "integer"), List.of(column("year")));++ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered("SELECT * FROM t", List.of(t)));+ }++ /** Identifiers match on word boundaries, so a longer name that contains one does not count. */+ @Test+ void aLongerIdentifierContainingTheColumnNameDoesNotCountAsAMention() {+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(+ "SELECT * FROM audit_events WHERE tenant_id = 'abc'", List.of(injectedTenantTable())));+ }++ /**+ * The column names itself in the SELECT list and the GROUP BY, and filters nothing - the case+ * the check exists to catch, and the one a query-wide text match lets through.+ */+ @Test+ void mentionOutsideAWhereClauseDoesNotCount() {+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(+ "SELECT tenant, count(*) FROM audit_events GROUP BY tenant",+ List.of(injectedTenantTable())));+ }++ /** A WHERE clause ends where the next one starts, so an ORDER BY mention is outside it. */+ @Test+ void mentionAfterTheWhereClauseEndsDoesNotCount() {+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(+ "SELECT * FROM audit_events WHERE year = 2026 ORDER BY tenant",+ List.of(injectedTenantTable())));+ }++ /** A string that happens to spell the column name constrains nothing. */+ @Test+ void mentionInsideAStringLiteralDoesNotCount() {+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(+ "SELECT * FROM audit_events WHERE note = 'tenant'", List.of(injectedTenantTable())));+ }++ /** Nor does one in a comment. */+ @Test+ void mentionInsideACommentDoesNotCount() {+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(+ "SELECT * FROM audit_events -- filter by tenant one day\n WHERE year = 2026",+ List.of(injectedTenantTable())));+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(+ "SELECT * FROM audit_events /* tenant */ WHERE year = 2026",+ List.of(injectedTenantTable())));+ }++ /** A subquery's WHERE is still a WHERE, and still constrains the read. */+ @Test+ void filterInASubqueryCounts() {+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(+ "SELECT tenant, count(*) FROM (SELECT * FROM audit_events WHERE tenant = 'abc') "+ + "GROUP BY tenant",+ List.of(injectedTenantTable())));+ }++ /** A predicate carrying its own subquery is not cut short at the inner parentheses. */+ @Test+ void aPredicateWithItsOwnSubqueryIsReadWhole() {+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(+ "SELECT * FROM audit_events WHERE year IN (SELECT y FROM years) AND tenant = 'abc'",+ List.of(injectedTenantTable())));+ }++ /** Double quotes mark an identifier in Athena, so a quoted column reference counts. */+ @Test+ void aQuotedColumnReferenceCounts() {+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(+ "SELECT * FROM audit_events WHERE \"tenant\" = 'abc'", List.of(injectedTenantTable())));+ }++ /** A column named after a clause keyword is quoted, and must not end the clause it sits in. */+ @Test+ void aQuotedIdentifierSpellingAClauseKeywordDoesNotEndTheClause() {+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(+ "SELECT * FROM audit_events WHERE \"limit\" = 10 AND tenant = 'abc'",+ List.of(injectedTenantTable())));+ }+}