Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Refactor custom query validation and enhance table reference collection

Bill Yange325d6645478
diff --git a/server/src/api/analytics/utils/customQueryValidation.test.ts b/server/src/api/analytics/utils/customQueryValidation.test.ts
new file mode 100644
index 000000000..b2248cd6f
--- /dev/null
+++ b/server/src/api/analytics/utils/customQueryValidation.test.ts
@@ -0,0 +1,69 @@
+import { describe, expect, it } from "vitest";
+import { validateScopedQuery } from "./customQueryValidation.js";
+
+const SCOPED_ONLY_ERROR = "Queries can only read from scoped_events";
+
+describe("validateScopedQuery β€” table reference scoping", () => {
+ it("allows reading from scoped_events", () => {
+ expect(validateScopedQuery("SELECT count(*) FROM scoped_events")).toBeNull();
+ });
+
+ it("allows CTEs and self-joins on scoped_events", () => {
+ expect(
+ validateScopedQuery(
+ "WITH t AS (SELECT user_id, count() c FROM scoped_events GROUP BY user_id) SELECT * FROM t"
+ )
+ ).toBeNull();
+ expect(
+ validateScopedQuery(
+ "SELECT a.user_id FROM scoped_events a JOIN scoped_events b ON a.user_id = b.user_id"
+ )
+ ).toBeNull();
+ });
+
+ it("does not flag commas in SELECT lists, GROUP BY, ORDER BY, or function args", () => {
+ expect(
+ validateScopedQuery(
+ "SELECT count(*), uniq(user_id) FROM scoped_events GROUP BY pathname ORDER BY pathname, count() LIMIT 10"
+ )
+ ).toBeNull();
+ });
+
+ it("blocks comma-join to a materialized view target table (RYB-015)", () => {
+ expect(
+ validateScopedQuery(
+ "SELECT sessions_mv_target.site_id FROM scoped_events, sessions_mv_target WHERE sessions_mv_target.site_id > 0 LIMIT 100"
+ )
+ ).toBe(SCOPED_ONLY_ERROR);
+ expect(
+ validateScopedQuery(
+ "SELECT pathname_hourly_mv_target.pathname FROM scoped_events, pathname_hourly_mv_target LIMIT 100"
+ )
+ ).toBe(SCOPED_ONLY_ERROR);
+ });
+
+ it("blocks comma-join without surrounding whitespace", () => {
+ expect(
+ validateScopedQuery("SELECT * FROM scoped_events,sessions_mv_target LIMIT 1")
+ ).toBe(SCOPED_ONLY_ERROR);
+ });
+
+ it("blocks an unauthorized table inside a subquery comma-join", () => {
+ expect(
+ validateScopedQuery(
+ "SELECT * FROM (SELECT * FROM scoped_events, sessions_mv_target) x LIMIT 1"
+ )
+ ).toBe(SCOPED_ONLY_ERROR);
+ });
+
+ it("still blocks plain FROM/JOIN to unauthorized tables", () => {
+ expect(validateScopedQuery("SELECT * FROM events")).toBe(SCOPED_ONLY_ERROR);
+ expect(
+ validateScopedQuery("SELECT * FROM scoped_events JOIN events ON 1=1")
+ ).toBe(SCOPED_ONLY_ERROR);
+ });
+
+ it("requires the query to reference scoped_events at all", () => {
+ expect(validateScopedQuery("SELECT 1")).toBe("Query must read from scoped_events");
+ });
+});
diff --git a/server/src/api/analytics/utils/customQueryValidation.ts b/server/src/api/analytics/utils/customQueryValidation.ts
index 6f873a8e9..b708f2cf6 100644
--- a/server/src/api/analytics/utils/customQueryValidation.ts
+++ b/server/src/api/analytics/utils/customQueryValidation.ts
@@ -77,18 +77,6 @@ const blockedFunctions = [
"viewIfPermitted",
] as const;
-const blockedTableNames = [
- "bot_events",
- "events",
- "hourly_events_by_site_mv",
- "hourly_events_by_site_mv_target",
- "information_schema",
- "monitor_events",
- "session_replay_events",
- "session_replay_metadata",
- "system",
-] as const;
-
function stripSqlLiteralsAndComments(query: string) {
let result = "";
let index = 0;
@@ -186,6 +174,112 @@ function getCteNames(query: string) {
return cteNames;
}
+// Identifiers that end a FROM clause's comma-separated table list. Once one of these
+// appears at the top paren level, later commas belong to another clause (GROUP BY,
+// ORDER BY, a UNIONed SELECT, …) rather than the table list.
+const fromClauseTerminators = new Set([
+ "where",
+ "prewhere",
+ "group",
+ "having",
+ "order",
+ "limit",
+ "settings",
+ "union",
+ "intersect",
+ "except",
+ "window",
+ "qualify",
+ "format",
+ "into",
+]);
+
+function isIdentifierStart(char: string) {
+ return /[A-Za-z_]/.test(char);
+}
+
+function isIdentifierChar(char: string) {
+ return /[A-Za-z0-9_.]/.test(char);
+}
+
+// Collect every directly-named table reference in the query. Covers the table after
+// each FROM/JOIN and every comma-separated entry in a FROM list (`FROM a, b`), which
+// a FROM/JOIN-keyword-only scan would miss. Subquery references (`FROM ( SELECT … )`)
+// are skipped here β€” their inner FROM/JOIN clauses are reached by this same scan.
+function collectTableReferences(query: string): string[] {
+ const references: string[] = [];
+ const length = query.length;
+
+ const readIdentifier = (start: number): [string, number] => {
+ let end = start;
+ while (end < length && isIdentifierChar(query[end])) {
+ end++;
+ }
+ return [query.slice(start, end), end];
+ };
+
+ const skipWhitespace = (index: number): number => {
+ while (index < length && /\s/.test(query[index])) {
+ index++;
+ }
+ return index;
+ };
+
+ // Record the table reference that follows a FROM / JOIN / comma when it is a plain
+ // identifier; subqueries and anything else are left to the surrounding scan.
+ const readReference = (index: number) => {
+ const start = skipWhitespace(index);
+ if (start < length && isIdentifierStart(query[start])) {
+ references.push(readIdentifier(start)[0]);
+ }
+ };
+
+ const keywordPattern = /\b(FROM|JOIN)\b/gi;
+ let match: RegExpExecArray | null;
+ while ((match = keywordPattern.exec(query)) !== null) {
+ const afterKeyword = match.index + match[0].length;
+
+ // A JOIN introduces exactly one table reference.
+ if (match[1].toLowerCase() === "join") {
+ readReference(afterKeyword);
+ continue;
+ }
+
+ // A FROM introduces a comma-separated table list. Read the first reference, then
+ // walk the clause tracking paren depth and pick up every top-level comma entry.
+ readReference(afterKeyword);
+
+ let depth = 0;
+ let index = afterKeyword;
+ while (index < length) {
+ const char = query[index];
+ if (char === "(") {
+ depth++;
+ index++;
+ } else if (char === ")") {
+ if (depth === 0) {
+ break; // a closing paren that ends an enclosing subquery β€” clause is done
+ }
+ depth--;
+ index++;
+ } else if (char === "," && depth === 0) {
+ readReference(index + 1);
+ index++;
+ } else if (depth === 0 && isIdentifierStart(char)) {
+ const [word, end] = readIdentifier(index);
+ if (fromClauseTerminators.has(word.toLowerCase())) {
+ break;
+ }
+ index = end; // an alias or join keyword β€” skip past it
+ } else {
+ index++;
+ }
+ }
+ }
+
+ return references;
+}
+
export function validateScopedQuery(query: string): string | null {
const normalizedQuery = normalizeCustomQuery(query);
const queryWithoutLiterals = stripSqlLiteralsAndComments(normalizedQuery);
@@ -229,25 +323,12 @@ export function validateScopedQuery(query: string): string | null {
return "scoped_events is reserved and cannot be redefined";
}
- for (const tableName of blockedTableNames) {
- if (new RegExp(`,\\s*${tableName}\\b`, "i").test(compactQuery)) {
- return "Queries can only read from scoped_events";
- }
- }
-
- // Match `FROM <identifier>` (requires whitespace, so `FROMx` won't match) and
- // `FROM(` / `FROM (` (a subquery, captured as undefined β†’ skipped). Every named
- // target must be scoped_events or a declared CTE.
- const tableReferencePattern = /\b(?:FROM|JOIN)(?:\s+([a-zA-Z_][a-zA-Z0-9_.]*)|\s*\()/gi;
- let match: RegExpExecArray | null;
- while ((match = tableReferencePattern.exec(compactQuery)) !== null) {
- const tableName = match[1];
- if (!tableName) {
- // Paren branch: a subquery, validated by subsequent FROM/JOIN matches.
- continue;
- }
-
- const normalizedTableName = tableName.toLowerCase();
+ // Every table reference must be scoped_events or a declared CTE. collectTableReferences
+ // walks the full FROM list, so comma-separated targets (`FROM scoped_events, other`) are
+ // validated too β€” a FROM/JOIN-keyword-only scan captured only the first table and let the
+ // rest through. Subqueries are validated by their own inner FROM/JOIN clauses.
+ for (const reference of collectTableReferences(compactQuery)) {
+ const normalizedTableName = reference.toLowerCase();
if (normalizedTableName !== "scoped_events" && !cteNames.has(normalizedTableName)) {
return "Queries can only read from scoped_events";
}
diff --git a/server/src/services/usageService.ts b/server/src/services/usageService.ts
index 218ef9e14..b59a42bc0 100644
--- a/server/src/services/usageService.ts
+++ b/server/src/services/usageService.ts
@@ -137,11 +137,6 @@ class UsageService {
},
stripeSubscriptions: Map<string, Stripe.Subscription>
): Promise<[number, string | null]> {
- // Special case for specific organizations
- if (orgData.name === "rybbit" || orgData.name === "Zam") {
- return [Infinity, this.getStartOfMonth()];
- }
-
// Resolve this org's Stripe subscription from the bulk snapshot (no per-org Stripe call),
// then layer in custom plan / override / AppSumo via the same priority rules as elsewhere.
const stripeSub = stripeSubscriptionInfoFromSnapshot(stripeSubscriptions, orgData.stripeCustomerId);
@@ -267,12 +262,7 @@ class UsageService {
const isOverLimit = eventCount > eventLimit;
let sendApproaching = false;
- if (
- !alreadyNotifiedApproaching &&
- !isOverLimit &&
- Number.isFinite(eventLimit) &&
- daysRemaining >= 2
- ) {
+ if (!alreadyNotifiedApproaching && !isOverLimit && Number.isFinite(eventLimit) && daysRemaining >= 2) {
const projected = daysElapsed >= 1 ? eventCount * (totalDaysInMonth / daysElapsed) : 0;
const trigger90 = eventCount >= eventLimit * 0.9;
const triggerProjection = daysElapsed >= 7 && projected >= eventLimit;
@@ -328,9 +318,7 @@ class UsageService {
}
}
} else {
- this.logger.warn(
- `No owners found for organization ${orgData.name}, skipping approaching-limit email`
- );
+ this.logger.warn(`No owners found for organization ${orgData.name}, skipping approaching-limit email`);
}
}