Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(athena): accept IN lists and check each table in the injected-partition filter (#5004)

Vũ Thế Huy008ddea05475
diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/PartitionProjection.java b/src/main/java/io/github/hectorvent/floci/services/glue/PartitionProjection.java
index b76ff9dd94..9dd1d4c707 100644
--- a/src/main/java/io/github/hectorvent/floci/services/glue/PartitionProjection.java
+++ b/src/main/java/io/github/hectorvent/floci/services/glue/PartitionProjection.java
@@ -5,8 +5,10 @@ import io.github.hectorvent.floci.services.glue.model.Column;
import io.github.hectorvent.floci.services.glue.model.Table;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
@@ -43,6 +45,15 @@ public final class PartitionProjection {
"(?:GROUP|HAVING|ORDER|WINDOW|LIMIT|OFFSET|FETCH|UNION|INTERSECT|EXCEPT)"),
Pattern.CASE_INSENSITIVE);
+ private static final Pattern CLAUSE_KEYWORD = Pattern.compile(
+ KEYWORD_BOUNDS.formatted(
+ "(?:SELECT|FROM|WHERE|GROUP|HAVING|ORDER|LIMIT|OFFSET|UNION|INTERSECT|EXCEPT|JOIN|ON|USING|WINDOW|QUALIFY|FETCH)"),
+ Pattern.CASE_INSENSITIVE);
+
+ private static final Set<String> TABLE_ALIAS_KEYWORDS = Set.of("where", "join", "inner", "left", "right",
+ "full", "cross", "on", "group", "having", "order", "limit", "offset", "union", "intersect",
+ "except", "window", "qualify", "fetch", "using", "natural", "lateral", "tablesample");
+
private PartitionProjection() {
}
@@ -111,17 +122,13 @@ public final class PartitionProjection {
*
* <p>An injected column has no generatable range: its values come from the query, so Athena
* requires a static equality condition on it in the {@code WHERE} clause and rejects the query
- * otherwise. Deciding whether a condition is a <em>static equality</em> one needs the query's
- * predicate tree, which is not available here, so this checks only whether the column is
- * mentioned inside a {@code WHERE} clause - any of them, including a subquery's. That is
- * strictly narrower than the real rule - a query filtering an injected column with, say, a
- * range condition is rejected by Athena and accepted here - but it never rejects a query Athena
- * would accept, and it catches the case that silently reads every partition.
+ * otherwise. Each source table alias is checked independently, and joins must qualify the
+ * partition column so a filter on one table cannot stand in for another table's filter.
*
- * <p>Scoping to the {@code WHERE} clause is what makes the check worth having. A mention
- * anywhere in the query text also matches the column's own appearance in a {@code SELECT} list
- * or a {@code GROUP BY}, so {@code SELECT tenant, count(*) FROM audit_events GROUP BY tenant}
- * would pass while filtering nothing at all - exactly the case this exists to catch.
+ * <p>Scoping to the {@code WHERE} clause is what makes the check useful. A mention anywhere in
+ * the query text also matches the column's own appearance in a {@code SELECT} list or a
+ * {@code GROUP BY}, so {@code SELECT tenant, count(*) FROM audit_events GROUP BY tenant} would
+ * pass while filtering nothing at all.
*/
public static void assertInjectedColumnsFiltered(String query, List<Table> tables) {
if (query == null || tables == null) {
@@ -129,13 +136,14 @@ public final class PartitionProjection {
}
// Literals and comments are not code: a column name inside either constrains nothing.
String code = maskLiteralsAndComments(query);
- List<String> whereClauses = null;
+ List<WhereScope> scopes = null;
for (Table table : tables) {
if (!enabled(table) || table.getName() == null || table.getPartitionKeys() == null) {
continue;
}
- // A table the query never names cannot be constrained by it, and must not fail it.
- if (!mentions(code, table.getName())) {
+ // A table the query never reads cannot be constrained by it, and must not fail it.
+ List<String> aliases = tableAliases(code, table.getName());
+ if (aliases.isEmpty()) {
continue;
}
for (Column key : table.getPartitionKeys()) {
@@ -146,10 +154,15 @@ public final class PartitionProjection {
if (!"injected".equalsIgnoreCase(type)) {
continue;
}
- if (whereClauses == null) {
- whereClauses = whereClauses(code);
+ if (scopes == null) {
+ scopes = whereScopes(code);
}
- if (whereClauses.stream().anyMatch(clause -> mentions(clause, key.getName()))) {
+ List<WhereScope> currentScopes = scopes;
+ boolean allFiltered = !aliases.isEmpty() && aliases.stream().allMatch(alias ->
+ currentScopes.stream().anyMatch(scope ->
+ isAliasFiltered(scope, table, alias, key.getName(), tables))
+ );
+ if (allFiltered) {
continue;
}
throw new AwsException("InvalidRequestException",
@@ -160,24 +173,85 @@ public final class PartitionProjection {
}
}
+ private static boolean isAliasFiltered(WhereScope scope, Table table, String alias, String column,
+ List<Table> tables) {
+ if (!tableAliases(scope.fromClause(), table.getName()).contains(alias)) {
+ return false;
+ }
+ Set<String> sourceCols = sourceColumns(scope, tables);
+ Set<String> sourceAliases = sourceAliases(scope, tables);
+ if (hasStaticEquality(scope.whereClause(), alias, column, false, sourceCols, sourceAliases)) {
+ return true;
+ }
+ long declaringSources = tables.stream()
+ .filter(other -> other != null && other.getName() != null && declaresColumn(other, column))
+ .mapToLong(other -> tableAliases(scope.fromClause(), other.getName()).size())
+ .sum();
+ if (declaringSources == 1
+ && hasStaticEquality(scope.whereClause(), alias, column, true, sourceCols, sourceAliases)) {
+ return true;
+ }
+ return false;
+ }
+
+ private static Set<String> sourceColumns(WhereScope scope, List<Table> tables) {
+ Set<String> columns = new HashSet<>();
+ for (Table table : tables) {
+ if (table == null || table.getName() == null) {
+ continue;
+ }
+ if (!tableAliases(scope.fromClause(), table.getName()).isEmpty()) {
+ if (table.getStorageDescriptor() != null && table.getStorageDescriptor().getColumns() != null) {
+ for (Column col : table.getStorageDescriptor().getColumns()) {
+ if (col != null && col.getName() != null && !col.getName().isBlank()) {
+ columns.add(col.getName().toLowerCase(Locale.ROOT));
+ }
+ }
+ }
+ if (table.getPartitionKeys() != null) {
+ for (Column key : table.getPartitionKeys()) {
+ if (key != null && key.getName() != null && !key.getName().isBlank()) {
+ columns.add(key.getName().toLowerCase(Locale.ROOT));
+ }
+ }
+ }
+ }
+ }
+ return columns;
+ }
+
+ private static Set<String> sourceAliases(WhereScope scope, List<Table> tables) {
+ Set<String> aliases = new HashSet<>();
+ for (Table table : tables) {
+ if (table == null || table.getName() == null) {
+ continue;
+ }
+ List<String> found = tableAliases(scope.fromClause(), table.getName());
+ if (!found.isEmpty()) {
+ aliases.add(unquoteIdentifier(table.getName()).toLowerCase(Locale.ROOT));
+ aliases.addAll(found);
+ }
+ }
+ return aliases;
+ }
+
+ private record WhereScope(String fromClause, String whereClause) {
+ }
+
/**
- * The text of every {@code WHERE} clause in {@code code}, a subquery's included.
- *
- * <p>A clause runs from the keyword to whatever ends it: a following clause keyword, the
- * {@code )} closing the subquery that holds it, a statement terminator, or the end of the
- * query. Parentheses opened inside the clause are part of it, so a predicate with its own
- * subquery is not cut short.
+ * Extracts every {@code WHERE} clause in {@code code} together with its preceding {@code FROM} clause.
*/
- private static List<String> whereClauses(String code) {
+ private static List<WhereScope> whereScopes(String code) {
Set<Integer> clauseStarts = new HashSet<>();
Matcher boundaries = CLAUSE_AFTER_WHERE.matcher(code);
while (boundaries.find()) {
clauseStarts.add(boundaries.start());
}
- List<String> clauses = new ArrayList<>();
+ List<WhereScope> scopes = new ArrayList<>();
Matcher where = WHERE_KEYWORD.matcher(code);
while (where.find()) {
+ int whereStart = where.start();
int start = where.end();
int depth = 0;
int i = start;
@@ -195,9 +269,84 @@ public final class PartitionProjection {
}
i++;
}
- clauses.add(code.substring(start, i));
+ String whereClause = code.substring(start, i);
+ String fromClause = findPrecedingFromClause(code, whereStart);
+ scopes.add(new WhereScope(fromClause, maskSubqueries(whereClause)));
+ }
+ return scopes;
+ }
+
+ /**
+ * Blanks out subqueries inside parentheses, so their internal predicates do not satisfy
+ * or conflict with the outer query's injected-partition filters.
+ */
+ private static String maskSubqueries(String clause) {
+ char[] chars = clause.toCharArray();
+ int i = 0;
+ while (i < chars.length) {
+ if (chars[i] == '(') {
+ int k = i + 1;
+ while (k < chars.length && Character.isWhitespace(chars[k])) {
+ k++;
+ }
+ boolean isSubquery = (k + 6 <= chars.length && clause.regionMatches(true, k, "SELECT", 0, 6)
+ && isWordBoundary(clause, k, 6))
+ || (k + 4 <= chars.length && clause.regionMatches(true, k, "WITH", 0, 4)
+ && isWordBoundary(clause, k, 4));
+ if (isSubquery) {
+ int depth = 1;
+ int subStart = i + 1;
+ int p = subStart;
+ while (p < chars.length && depth > 0) {
+ if (chars[p] == '(') {
+ depth++;
+ } else if (chars[p] == ')') {
+ depth--;
+ }
+ p++;
+ }
+ int subEnd = depth == 0 ? p - 1 : chars.length;
+ for (int m = subStart; m < subEnd; m++) {
+ if (chars[m] != '\n') {
+ chars[m] = ' ';
+ }
+ }
+ i = subEnd;
+ continue;
+ }
+ }
+ i++;
+ }
+ return new String(chars);
+ }
+
+ private static String findPrecedingFromClause(String code, int whereStart) {
+ int depth = 0;
+ for (int i = whereStart - 1; i >= 0; i--) {
+ char c = code.charAt(i);
+ if (c == ')') {
+ depth++;
+ } else if (c == '(') {
+ if (depth == 0) {
+ break;
+ }
+ depth--;
+ } else if (depth == 0) {
+ if (c == ';') {
+ break;
+ }
+ if (code.regionMatches(true, i, "FROM", 0, 4) && isWordBoundary(code, i, 4)) {
+ return code.substring(i, whereStart);
+ }
+ if ((code.regionMatches(true, i, "SELECT", 0, 6) && isWordBoundary(code, i, 6))
+ || (code.regionMatches(true, i, "UNION", 0, 5) && isWordBoundary(code, i, 5))
+ || (code.regionMatches(true, i, "INTERSECT", 0, 9) && isWordBoundary(code, i, 9))
+ || (code.regionMatches(true, i, "EXCEPT", 0, 6) && isWordBoundary(code, i, 6))) {
+ break;
+ }
+ }
}
- return clauses;
+ return "";
}
/**
@@ -238,7 +387,8 @@ public final class PartitionProjection {
continue;
}
for (int k = i; k < end; k++) {
- if (chars[k] != '\n') {
+ boolean literalBoundary = chars[i] == '\'' && (k == i || k == end - 1 && chars[k] == '\'');
+ if (chars[k] != '\n' && !literalBoundary) {
chars[k] = ' ';
}
}
@@ -247,9 +397,453 @@ public final class PartitionProjection {
return new String(chars);
}
- private static boolean mentions(String code, String identifier) {
- return Pattern.compile("(?<![A-Za-z0-9_])" + Pattern.quote(identifier) + "(?![A-Za-z0-9_])",
- Pattern.CASE_INSENSITIVE).matcher(code).find();
+ private static boolean declaresColumn(Table table, String column) {
+ List<Column> declared = new ArrayList<>();
+ if (table.getStorageDescriptor() != null && table.getStorageDescriptor().getColumns() != null) {
+ declared.addAll(table.getStorageDescriptor().getColumns());
+ }
+ if (table.getPartitionKeys() != null) {
+ declared.addAll(table.getPartitionKeys());
+ }
+ return declared.stream().anyMatch(c -> c != null && column.equalsIgnoreCase(c.getName()));
+ }
+
+ /**
+ * The alias, or the table's own name, under which each {@code FROM}, {@code JOIN} or comma-separated
+ * source naming {@code tableName} appears. The name must end at an identifier boundary, so
+ * {@code events_daily} is not a reference to {@code events}.
+ */
+ private static List<String> tableAliases(String code, String tableName) {
+ String identifier = identifierPattern(tableName);
+ String qualifiedName = "(?:" + IDENTIFIER + "\\s*\\.\\s*)*" + identifier + "(?![A-Za-z0-9_$])";
+ String aliasPart = "(?:\\s+(?:(AS)\\s+)?(" + IDENTIFIER + "))?";
+ Pattern keywordSource = Pattern.compile("(?i)" + KEYWORD_BOUNDS.formatted("(?:FROM|JOIN)") + "\\s+"
+ + qualifiedName + aliasPart);
+ Pattern commaSource = Pattern.compile("(?i),\\s*" + qualifiedName + aliasPart);
+ List<String> aliases = new ArrayList<>();
+ Matcher matcher = keywordSource.matcher(code);
+ while (matcher.find()) {
+ boolean hasAs = matcher.group(1) != null;
+ aliases.add(sourceName(matcher.group(2), hasAs, tableName));
+ }
+ matcher = commaSource.matcher(code);
+ while (matcher.find()) {
+ if (inFromList(code, matcher.start())) {
+ boolean hasAs = matcher.group(1) != null;
+ aliases.add(sourceName(matcher.group(2), hasAs, tableName));
+ }
+ }
+ return aliases;
+ }
+
+ private static String sourceName(String alias, boolean hasAs, String tableName) {
+ if (alias == null) {
+ return unquoteIdentifier(tableName).toLowerCase(Locale.ROOT);
+ }
+ boolean quoted = isQuoted(alias);
+ if (quoted || hasAs) {
+ return unquoteIdentifier(alias).toLowerCase(Locale.ROOT);
+ }
+ if (TABLE_ALIAS_KEYWORDS.contains(alias.toLowerCase(Locale.ROOT))) {
+ return unquoteIdentifier(tableName).toLowerCase(Locale.ROOT);
+ }
+ return unquoteIdentifier(alias).toLowerCase(Locale.ROOT);
+ }
+
+ private static boolean isQuoted(String identifier) {
+ return identifier != null && identifier.length() >= 2
+ && ((identifier.startsWith("\"") && identifier.endsWith("\""))
+ || (identifier.startsWith("`") && identifier.endsWith("`")));
+ }
+
+ /** True when the nearest clause keyword before {@code position} is {@code FROM} or {@code JOIN}. */
+ private static boolean inFromList(String code, int position) {
+ int depth = 0;
+ Map<Integer, String> clauseAtDepth = new HashMap<>();
+ Matcher kwMatcher = CLAUSE_KEYWORD.matcher(code);
+ int cursor = 0;
+ while (kwMatcher.find() && kwMatcher.start() < position) {
+ for (int i = cursor; i < kwMatcher.start(); i++) {
+ char c = code.charAt(i);
+ if (c == '(') {
+ depth++;
+ } else if (c == ')' && depth > 0) {
+ clauseAtDepth.remove(depth);
+ depth--;
+ } else if (c == ';' && depth == 0) {
+ clauseAtDepth.clear();
+ }
+ }
+ cursor = kwMatcher.start();
+ String kw = kwMatcher.group().toUpperCase(Locale.ROOT);
+ if ("FROM".equals(kw) || "JOIN".equals(kw) || "ON".equals(kw) || "USING".equals(kw)) {
+ clauseAtDepth.put(depth, "FROM");
+ } else {
+ clauseAtDepth.put(depth, kw);
+ }
+ cursor = kwMatcher.end();
+ }
+ for (int i = cursor; i < position; i++) {
+ char c = code.charAt(i);
+ if (c == '(') {
+ depth++;
+ } else if (c == ')' && depth > 0) {
+ clauseAtDepth.remove(depth);
+ depth--;
+ } else if (c == ';' && depth == 0) {
+ clauseAtDepth.clear();
+ }
+ }
+ return "FROM".equals(clauseAtDepth.get(depth));
+ }
+
+ private static final String IDENTIFIER = "(?:[A-Za-z_][A-Za-z0-9_$]*|\"(?:\"\"|[^\"])+\"|`[^`]+`)";
+
+ private static String identifierPattern(String identifier) {
+ String unquoted = unquoteIdentifier(identifier);
+ String doubleQuoted = "\"" + unquoted.replace("\"", "\"\"") + "\"";
+ String backquoted = "`" + unquoted + "`";
+ return "(?:" + Pattern.quote(unquoted) + "|" + Pattern.quote(doubleQuoted) + "|"
+ + Pattern.quote(backquoted) + ")";
+ }
+
+ private static String unquoteIdentifier(String identifier) {
+ if (identifier.length() >= 2 && identifier.startsWith("\"") && identifier.endsWith("\"")) {
+ return identifier.substring(1, identifier.length() - 1).replace("\"\"", "\"");
+ }
+ if (identifier.length() >= 2 && identifier.startsWith("`") && identifier.endsWith("`")) {
+ return identifier.substring(1, identifier.length() - 1);
+ }
+ return identifier;
+ }
+
+ private static boolean hasStaticEquality(String clause, String alias, String column, boolean allowUnqualified,
+ Set<String> sourceColumns, Set<String> sourceAliases) {
+ String qualified = identifierPattern(alias) + "\\s*\\.\\s*" + identifierPattern(column);
+ String left = allowUnqualified ? "(?:" + qualified + "|" + identifierPattern(column) + ")" : qualified;
+ Pattern colPattern = Pattern.compile("(?<![A-Za-z0-9_$])" + left + "(?![A-Za-z0-9_$])",
+ Pattern.CASE_INSENSITIVE);
+ Matcher m = colPattern.matcher(clause);
+ while (m.find()) {
+ int colStart = m.start();
+ int colEnd = m.end();
+
+ // 1. IN list: col [NOT] IN (...) or NOT col IN (...)
+ int afterCol = colEnd;
+ while (afterCol < clause.length() && Character.isWhitespace(clause.charAt(afterCol))) {
+ afterCol++;
+ }
+ boolean isNotIn = isPrecededByNot(clause, colStart);
+ if (afterCol + 3 <= clause.length() && clause.regionMatches(true, afterCol, "NOT", 0, 3)
+ && isWordBoundary(clause, afterCol, 3)) {
+ isNotIn = true;
+ afterCol += 3;
+ while (afterCol < clause.length() && Character.isWhitespace(clause.charAt(afterCol))) {
+ afterCol++;
+ }
+ }
+ if (afterCol + 2 <= clause.length() && clause.regionMatches(true, afterCol, "IN", 0, 2)
+ && isWordBoundary(clause, afterCol, 2)) {
+ if (!isNotIn) {
+ int afterIn = afterCol + 2;
+ while (afterIn < clause.length() && Character.isWhitespace(clause.charAt(afterIn))) {
+ afterIn++;
+ }
+ if (afterIn < clause.length() && clause.charAt(afterIn) == '(') {
+ int parenStart = afterIn;
+ int parenEnd = findMatchingParen(clause, parenStart);
+ if (parenEnd != -1) {
+ String listContent = clause.substring(parenStart + 1, parenEnd);
+ List<String> items = splitList(listContent);
+ if (!items.isEmpty() && items.size() <= 1000
+ && items.stream().allMatch(item ->
+ isValidValue(item, sourceColumns, sourceAliases))) {
+ return true;
+ }
+ }
+ }
+ }
+ continue;
+ }
+
+ // 2. Equality on the left: col = value
+ int p = colEnd;
+ while (p < clause.length()) {
+ char c = clause.charAt(p);
+ if (Character.isWhitespace(c) || c == ')') {
+ p++;
+ } else {
+ break;
+ }
+ }
+ if (p < clause.length() && clause.charAt(p) == '=') {
+ if (isSingleEquals(clause, p, colEnd)) {
+ int valStart = p + 1;
+ int depth = 0;
+ int k = valStart;
+ while (k < clause.length()) {
+ char c = clause.charAt(k);
+ if (c == '(') {
+ depth++;
+ } else if (c == ')') {
+ if (depth == 0) {
+ break;
+ }
+ depth--;
+ } else if (depth == 0) {
+ if (c == ';') {
+ break;
+ }
+ if (isKeywordAt(clause, k, "AND") || isKeywordAt(clause, k, "OR")) {
+ break;
+ }
+ }
+ k++;
+ }
+ String valExpr = clause.substring(valStart, k).trim();
+ if (isValidValue(valExpr, sourceColumns, sourceAliases)) {
+ return true;
+ }
+ }
+ }
+
+ // 3. Equality on the right: value = col
+ int q = colStart - 1;
+ while (q >= 0) {
+ char c = clause.charAt(q);
+ if (Character.isWhitespace(c) || c == '(') {
+ q--;
+ } else {
+ break;
+ }
+ }
+ if (q >= 0 && clause.charAt(q) == '=') {
+ if (isSingleEquals(clause, q, 0)) {
+ int valEnd = q;
+ int depth = 0;
+ int j = q - 1;
+ while (j >= 0) {
+ char c = clause.charAt(j);
+ if (c == ')') {
+ depth++;
+ } else if (c == '(') {
+ if (depth == 0) {
+ break;
+ }
+ depth--;
+ } else if (depth == 0) {
+ if (c == ';') {
+ break;
+ }
+ if (isKeywordBefore(clause, j, "AND") || isKeywordBefore(clause, j, "OR")) {
+ break;
+ }
+ }
+ j--;
+ }
+ int valStart = j + 1;
+ String valExpr = clause.substring(valStart, valEnd).trim();
+ if (isValidValue(valExpr, sourceColumns, sourceAliases)) {
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ private static boolean isValidValue(String expr, Set<String> sourceColumns, Set<String> sourceAliases) {
+ if (expr == null) {
+ return false;
+ }
+ String stripped = stripParens(expr).trim();
+ if (stripped.isEmpty()) {
+ return false;
+ }
+ return !namesSourceColumn(expr, sourceColumns, sourceAliases);
+ }
+
+ private static boolean namesSourceColumn(String expr, Set<String> sourceColumns, Set<String> sourceAliases) {
+ if (expr == null || expr.isBlank()) {
+ return false;
+ }
+ for (String col : sourceColumns) {
+ Pattern pattern = Pattern.compile("(?<![A-Za-z0-9_$])" + identifierPattern(col)
+ + "(?![A-Za-z0-9_$])(?!\\s*\\()", Pattern.CASE_INSENSITIVE);
+ Matcher matcher = pattern.matcher(expr);
+ while (matcher.find()) {
+ int matchStart = matcher.start();
+ int b = matchStart - 1;
+ while (b >= 0 && Character.isWhitespace(expr.charAt(b))) {
+ b--;
+ }
+ if (b >= 1 && expr.regionMatches(true, b - 1, "AS", 0, 2)
+ && isWordBoundary(expr, b - 1, 2)) {
+ continue;
+ }
+ return true;
+ }
+ }
+ Pattern dotChainPattern = Pattern.compile("(?<![A-Za-z0-9_$])" + IDENTIFIER
+ + "(?:\\s*\\.\\s*" + IDENTIFIER + ")+", Pattern.CASE_INSENSITIVE);
+ Matcher dotMatcher = dotChainPattern.matcher(expr);
+ while (dotMatcher.find()) {
+ int end = dotMatcher.end();
+ while (end < expr.length() && Character.isWhitespace(expr.charAt(end))) {
+ end++;
+ }
+ if (end < expr.length() && expr.charAt(end) == '(') {
+ continue;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ private static boolean isSingleEquals(String text, int equalsIndex, int minBeforeIndex) {
+ int before = equalsIndex - 1;
+ while (before >= minBeforeIndex && Character.isWhitespace(text.charAt(before))) {
+ before--;
+ }
+ if (before >= 0) {
+ char b = text.charAt(before);
+ if (b == '!' || b == '<' || b == '>' || b == '=') {
+ return false;
+ }
+ }
+ int after = equalsIndex + 1;
+ while (after < text.length() && Character.isWhitespace(text.charAt(after))) {
+ after++;
+ }
+ if (after < text.length()) {
+ char a = text.charAt(after);
+ if (a == '=' || a == '<' || a == '>') {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static int findMatchingParen(String text, int openParenIndex) {
+ int depth = 1;
+ int i = openParenIndex + 1;
+ while (i < text.length() && depth > 0) {
+ char c = text.charAt(i);
+ if (c == '(') {
+ depth++;
+ } else if (c == ')') {
+ depth--;
+ }
+ i++;
+ }
+ return depth == 0 ? i - 1 : -1;
+ }
+
+ private static boolean isPrecededByNot(String text, int colStart) {
+ int j = colStart - 1;
+ while (j >= 0 && (Character.isWhitespace(text.charAt(j)) || text.charAt(j) == '(')) {
+ j--;
+ }
+ if (j < 2) {
+ return false;
+ }
+ return text.regionMatches(true, j - 2, "NOT", 0, 3) && isWordBoundary(text, j - 2, 3);
+ }
+
+ private static boolean isKeywordAt(String text, int index, String keyword) {
+ int len = keyword.length();
+ if (index + len > text.length()) {
+ return false;
+ }
+ if (!text.regionMatches(true, index, keyword, 0, len)) {
+ return false;
+ }
+ return isWordBoundary(text, index, len);
+ }
+
+ private static boolean isKeywordBefore(String text, int index, String keyword) {
+ int len = keyword.length();
+ int end = index;
+ while (end >= 0 && Character.isWhitespace(text.charAt(end))) {
+ end--;
+ }
+ if (end + 1 < len) {
+ return false;
+ }
+ int start = end - len + 1;
+ return text.regionMatches(true, start, keyword, 0, len) && isWordBoundary(text, start, len);
+ }
+
+ private static String stripParens(String expr) {
+ if (expr == null) {
+ return "";
+ }
+ String trimmed = expr.trim();
+ while (trimmed.startsWith("(") && trimmed.endsWith(")")) {
+ int depth = 0;
+ boolean matching = false;
+ for (int i = 0; i < trimmed.length(); i++) {
+ char c = trimmed.charAt(i);
+ if (c == '(') {
+ depth++;
+ } else if (c == ')') {
+ depth--;
+ if (depth == 0) {
+ matching = (i == trimmed.length() - 1);
+ break;
+ }
+ }
+ }
+ if (!matching) {
+ break;
+ }
+ trimmed = trimmed.substring(1, trimmed.length() - 1).trim();
+ }
+ return trimmed;
+ }
+
+ private static List<String> splitList(String content) {
+ List<String> items = new ArrayList<>();
+ int depth = 0;
+ int start = 0;
+ for (int i = 0; i < content.length(); i++) {
+ char c = content.charAt(i);
+ if (c == '(') {
+ depth++;
+ } else if (c == ')') {
+ if (depth > 0) {
+ depth--;
+ }
+ } else if (c == ',' && depth == 0) {
+ String item = content.substring(start, i).trim();
+ if (!item.isEmpty()) {
+ items.add(item);
+ }
+ start = i + 1;
+ }
+ }
+ String last = content.substring(start).trim();
+ if (!last.isEmpty()) {
+ items.add(last);
+ }
+ return items;
+ }
+
+ private static boolean isWordBoundary(String text, int start, int length) {
+ if (start > 0) {
+ char before = text.charAt(start - 1);
+ if (Character.isLetterOrDigit(before) || before == '_' || before == '$' || before == '"' || before == '`') {
+ return false;
+ }
+ }
+ int end = start + length;
+ if (end < text.length()) {
+ char after = text.charAt(end);
+ if (Character.isLetterOrDigit(after) || after == '_' || after == '$' || after == '"' || after == '`') {
+ return false;
+ }
+ }
+ return true;
}
private static String stripTrailingSlash(String value) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/glue/PartitionProjectionTest.java b/src/test/java/io/github/hectorvent/floci/services/glue/PartitionProjectionTest.java
index 052fcb971e..fdeead1b64 100644
--- a/src/test/java/io/github/hectorvent/floci/services/glue/PartitionProjectionTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/glue/PartitionProjectionTest.java
@@ -6,6 +6,7 @@ import io.github.hectorvent.floci.services.glue.model.StorageDescriptor;
import io.github.hectorvent.floci.services.glue.model.Table;
import org.junit.jupiter.api.Test;
+import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -97,8 +98,11 @@ class PartitionProjectionTest {
// ---- injected columns ----
private Table injectedTenantTable() {
- return table("audit_events", projecting("projection.tenant.type", "injected"),
+ Table t = table("audit_events", projecting("projection.tenant.type", "injected"),
List.of(column("tenant")));
+ t.getStorageDescriptor().setColumns(List.of(column("id"), column("suffix"), column("other"),
+ column("other_tenant")));
+ return t;
}
@Test
@@ -117,6 +121,85 @@ class PartitionProjectionTest {
"SELECT * FROM audit_events WHERE tenant = 'abc'", List.of(injectedTenantTable())));
}
+ @Test
+ void inListOfLiteralsConstrainsTheInjectedColumn() {
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant IN ('abc', 'def')", List.of(injectedTenantTable())));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = 'abc' OR tenant = 'def'", List.of(injectedTenantTable())));
+ }
+
+ @Test
+ void aTableNameThatOnlyPrefixesAnotherTableIsNotAReferenceToIt() {
+ Table daily = table("events_daily", Map.of(), List.of());
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM events_daily",
+ List.of(table("events", projecting("projection.tenant.type", "injected"),
+ List.of(column("tenant"))), daily)));
+ }
+
+ @Test
+ void anUnqualifiedColumnIsAcceptedInAJoinWhenOnlyOneSourceDeclaresIt() {
+ Table orders = table("orders", Map.of(), List.of());
+ orders.getStorageDescriptor().setColumns(List.of(column("id")));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events a JOIN orders o ON a.id = o.id WHERE tenant = 'abc'",
+ List.of(injectedTenantTable(), orders)));
+ Table ordersWithTenant = table("orders", Map.of(), List.of());
+ ordersWithTenant.getStorageDescriptor().setColumns(List.of(column("tenant")));
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events a JOIN orders o ON a.id = o.id WHERE tenant = 'abc'",
+ List.of(injectedTenantTable(), ordersWithTenant)));
+ }
+
+ @Test
+ void aCommaSeparatedSourceNeedsItsOwnFilter() {
+ Table orders = table("orders", Map.of(), List.of());
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM orders o, audit_events a WHERE o.id = 1", List.of(injectedTenantTable(), orders)));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM orders o, audit_events a WHERE a.tenant = 'abc'",
+ List.of(injectedTenantTable(), orders)));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT o.id, audit_events FROM orders o WHERE o.id = 1", List.of(injectedTenantTable(), orders)));
+ }
+
+ @Test
+ void aConstantInParenthesesIsStillAStaticEquality() {
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = ('abc')", List.of(injectedTenantTable())));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = (('abc'))", List.of(injectedTenantTable())));
+ }
+
+ @Test
+ void aComputedValueIsNotAStaticEquality() {
+ for (String predicate : List.of("tenant = ('abc' || suffix)", "tenant = 'abc' || suffix",
+ "tenant = ('abc') || suffix", "tenant = 1 + other", "tenant = 'abc'::varchar || suffix")) {
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE " + predicate, List.of(injectedTenantTable())), predicate);
+ }
+ }
+
+ @Test
+ void joinKeywordsAfterTheTableAreNotTakenForItsAlias() {
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events NATURAL JOIN other_table WHERE tenant = 'abc'",
+ List.of(injectedTenantTable())));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events TABLESAMPLE BERNOULLI (10) WHERE tenant = 'abc'",
+ List.of(injectedTenantTable())));
+ }
+
+ @Test
+ void notInAndSubqueryInListsDoNotConstrainTheInjectedColumn() {
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant NOT IN ('abc')", List.of(injectedTenantTable())));
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant IN (SELECT tenant FROM other_table)",
+ List.of(injectedTenantTable())));
+ }
+
/** A table the query never names must not fail it. */
@Test
void unrelatedQueryAgainstAnotherTableIsUnaffected() {
@@ -207,4 +290,177 @@ class PartitionProjectionTest {
"SELECT * FROM audit_events WHERE \"limit\" = 10 AND tenant = 'abc'",
List.of(injectedTenantTable())));
}
+
+ @Test
+ void injectedFiltersMustApplyToEveryJoinedTableAlias() {
+ Table second = table("payments", projecting("projection.tenant.type", "injected"),
+ List.of(column("tenant")));
+
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events a JOIN payments p ON a.id = p.id WHERE a.tenant = 'abc'",
+ List.of(injectedTenantTable(), second)));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events a JOIN payments p ON a.id = p.id "
+ + "WHERE a.tenant = 'abc' AND p.tenant = 'xyz'",
+ List.of(injectedTenantTable(), second)));
+ }
+
+ @Test
+ void injectedFilterMustUseStaticEquality() {
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant > 'abc'", List.of(injectedTenantTable())));
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant >= 'abc'", List.of(injectedTenantTable())));
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = other_tenant", List.of(injectedTenantTable())));
+ }
+
+ @Test
+ void commaSeparatedSourceAfterJoinConditionIsChecked() {
+ Table orders = table("orders", Map.of(), List.of());
+ Table payments = table("payments", Map.of(), List.of());
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM orders o JOIN payments p ON o.id = p.id, audit_events a WHERE o.id = 1",
+ List.of(injectedTenantTable(), orders, payments)));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM orders o JOIN payments p ON o.id = p.id, audit_events a WHERE a.tenant = 'abc'",
+ List.of(injectedTenantTable(), orders, payments)));
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM orders o JOIN payments p USING (id), audit_events a WHERE o.id = 1",
+ List.of(injectedTenantTable(), orders, payments)));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM orders o JOIN payments p USING (id), audit_events a WHERE a.tenant = 'abc'",
+ List.of(injectedTenantTable(), orders, payments)));
+ }
+
+ @Test
+ void nestedQueryDeclaringSameColumnDoesNotCreateAmbiguityInOuterFilter() {
+ Table other = table("other_table", Map.of(), List.of());
+ other.getStorageDescriptor().setColumns(List.of(column("tenant")));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = 'abc' AND id IN (SELECT id FROM other_table WHERE foo = 1)",
+ List.of(injectedTenantTable(), other)));
+ }
+
+ @Test
+ void quotedAliasIsPreservedAndMatched() {
+ Table orders = table("orders", Map.of(), List.of());
+ orders.getStorageDescriptor().setColumns(List.of(column("tenant")));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events AS \"order\" JOIN orders o ON \"order\".id = o.id "
+ + "WHERE \"order\".tenant = 'abc'",
+ List.of(injectedTenantTable(), orders)));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events \"order\" JOIN orders o ON \"order\".id = o.id "
+ + "WHERE \"order\".tenant = 'abc'",
+ List.of(injectedTenantTable(), orders)));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events AS `order` JOIN orders o ON `order`.id = o.id "
+ + "WHERE `order`.tenant = 'abc'",
+ List.of(injectedTenantTable(), orders)));
+ }
+
+ @Test
+ void arbitraryParenthesesDepthAroundConstantIsAccepted() {
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = ((('abc')))", List.of(injectedTenantTable())));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = (((('abc'))))", List.of(injectedTenantTable())));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE ((('abc'))) = tenant", List.of(injectedTenantTable())));
+ }
+
+ @Test
+ void inListBoundaryEnforcesOneThousandItemLimit() {
+ List<String> thousandItems = new ArrayList<>();
+ for (int i = 0; i < 1000; i++) {
+ thousandItems.add("'val" + i + "'");
+ }
+ String validInListQuery = "SELECT * FROM audit_events WHERE tenant IN ("
+ + String.join(", ", thousandItems) + ")";
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ validInListQuery, List.of(injectedTenantTable())));
+
+ thousandItems.add("'val1000'");
+ String exceededInListQuery = "SELECT * FROM audit_events WHERE tenant IN ("
+ + String.join(", ", thousandItems) + ")";
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ exceededInListQuery, List.of(injectedTenantTable())));
+ }
+
+ @Test
+ void nestedFilterInSubqueryDoesNotSatisfyOuterTableCheck() {
+ Table other = table("other_table", Map.of(), List.of());
+ other.getStorageDescriptor().setColumns(List.of(column("tenant")));
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE id IN (SELECT id FROM other_table WHERE tenant = 'abc')",
+ List.of(injectedTenantTable(), other)));
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE id IN (SELECT id FROM other_table WHERE audit_events.tenant = 'abc')",
+ List.of(injectedTenantTable(), other)));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = 'abc' AND id IN (SELECT id FROM other_table WHERE tenant = 'xyz')",
+ List.of(injectedTenantTable(), other)));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM other_table WHERE id IN (SELECT id FROM audit_events WHERE tenant = 'abc')",
+ List.of(injectedTenantTable(), other)));
+ }
+
+ @Test
+ void expressionsEvaluatedWithoutReadingDataAreAccepted() {
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = lower('ABC')", List.of(injectedTenantTable())));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = 'ab' || 'c'", List.of(injectedTenantTable())));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = concat('a', 'b')", List.of(injectedTenantTable())));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = CAST(123 AS varchar)", List.of(injectedTenantTable())));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = ('ab' || 'c')", List.of(injectedTenantTable())));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE lower('ABC') = tenant", List.of(injectedTenantTable())));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant IN (lower('ABC'), 'def')", List.of(injectedTenantTable())));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant IN (concat('a', 'b'), 'c')",
+ List.of(injectedTenantTable())));
+ }
+
+ @Test
+ void functionNameOrCastTypeMatchingSourceColumnIsNotTakenForAColumnReference() {
+ Table tableWithCollidingCols = table("audit_events", projecting("projection.tenant.type", "injected"),
+ List.of(column("tenant")));
+ tableWithCollidingCols.getStorageDescriptor().setColumns(List.of(
+ column("id"), column("lower"), column("varchar"), column("suffix")));
+
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = lower('ABC')", List.of(tableWithCollidingCols)));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = CAST(123 AS varchar)", List.of(tableWithCollidingCols)));
+
+ // A column reference inside the function must still be rejected
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = lower(suffix)", List.of(tableWithCollidingCols)));
+ }
+
+ @Test
+ void cteColumnValueIsNotAcceptedAsAStaticFilter() {
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "WITH x AS (SELECT 1 AS key) SELECT * FROM audit_events a JOIN x ON a.id = x.key "
+ + "WHERE a.tenant = x.key", List.of(injectedTenantTable())));
+ }
+
+ @Test
+ void multiPartFunctionCallsAreAcceptedAsStaticFilters() {
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = catalog.schema.fn('abc')",
+ List.of(injectedTenantTable())));
+ assertDoesNotThrow(() -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = schema.fn('abc')",
+ List.of(injectedTenantTable())));
+ assertThrows(AwsException.class, () -> PartitionProjection.assertInjectedColumnsFiltered(
+ "SELECT * FROM audit_events WHERE tenant = catalog.schema.fn(suffix)",
+ List.of(injectedTenantTable())));
+ }
}