Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

feat: apply isMultiple again and add filters in Events page

Pedro Ladeiraa0db5f6dd7e3
diff --git a/apps/web/lib/analytics/build-advanced-filters.ts b/apps/web/lib/analytics/build-advanced-filters.ts
index 0dde686ab8..e3909201b7 100644
--- a/apps/web/lib/analytics/build-advanced-filters.ts
+++ b/apps/web/lib/analytics/build-advanced-filters.ts
@@ -6,8 +6,6 @@ export interface AdvancedFilter {
values: string[];
}
-// All fields that support advanced filtering with operators (IN, NOT IN)
-// These fields will be passed via the JSON filters array to Tinybird
const SUPPORTED_FIELDS = [
"country",
"city",
@@ -24,10 +22,6 @@ const SUPPORTED_FIELDS = [
"utm_campaign",
"utm_term",
"utm_content",
- "domain",
- "tagIds",
- "folderId",
- "root",
"saleType",
] as const;
diff --git a/apps/web/lib/analytics/get-analytics.ts b/apps/web/lib/analytics/get-analytics.ts
index 6b8dfa66da..f2482ab10b 100644
--- a/apps/web/lib/analytics/get-analytics.ts
+++ b/apps/web/lib/analytics/get-analytics.ts
@@ -45,7 +45,7 @@ export const getAnalytics = async (params: AnalyticsFilters) => {
// 4. no custom start or end date is provided
// 5. no other dimensional filters are applied
if (
- (linkIds) &&
+ linkIds &&
groupBy === "count" &&
interval === "all" &&
!start &&
@@ -54,24 +54,6 @@ export const getAnalytics = async (params: AnalyticsFilters) => {
(filter) => !params[filter as keyof AnalyticsFilters],
)
) {
- const columns =
- event === "composite"
- ? `clicks, leads, sales, saleAmount`
- : event === "sales"
- ? `sales, saleAmount`
- : `${event}`;
-
- // // Handle single linkId
- // if (linkId) {
- // const response = await conn.execute(
- // `SELECT ${columns} FROM Link WHERE id = ?`,
- // [linkId],
- // );
-
- // return analyticsResponse["count"].parse(response.rows[0]);
- // }
-
- // Handle multiple linkIds with aggregation
if (linkIds && linkIds.length > 0) {
const linkIdsToFilter = linkIds.map(() => "?").join(",");
const aggregateColumns =
@@ -90,9 +72,7 @@ export const getAnalytics = async (params: AnalyticsFilters) => {
}
}
- if (groupBy === "trigger") {
- groupBy = "triggers";
- }
+ if (groupBy === "trigger") groupBy = "triggers";
const { startDate, endDate, granularity } = getStartEndDates({
interval,
@@ -146,30 +126,25 @@ export const getAnalytics = async (params: AnalyticsFilters) => {
const metadataFilters = queryParser(query) || [];
const advancedFilters = buildAdvancedFilters({
+ ...params,
country: countryForPipe,
trigger: triggerForPipe,
- city: params.city,
- continent: params.continent,
- device: params.device,
- browser: params.browser,
- os: params.os,
- referer: params.referer,
- refererUrl: params.refererUrl,
- url: params.url,
- utm_source: params.utm_source,
- utm_medium: params.utm_medium,
- utm_campaign: params.utm_campaign,
- utm_term: params.utm_term,
- utm_content: params.utm_content,
- domain: params.domain,
- tagIds: params.tagIds,
- folderId: params.folderId,
- root: params.root,
- saleType: params.saleType,
});
const allFilters = [...metadataFilters, ...advancedFilters];
+ const domainParam = params.domain?.values;
+ const domainOperator = params.domain?.sqlOperator === 'NOT IN' ? 'NOT IN' : 'IN';
+
+ const tagIdsParam = params.tagIds?.values;
+ const tagIdsOperator = params.tagIds?.sqlOperator === 'NOT IN' ? 'NOT IN' : 'IN';
+
+ const folderIdParam = params.folderId?.values;
+ const folderIdOperator = params.folderId?.sqlOperator === 'NOT IN' ? 'NOT IN' : 'IN';
+
+ const rootParam = params.root?.values;
+ const rootOperator = params.root?.sqlOperator === 'NOT IN' ? 'NOT IN' : 'IN';
+
const tinybirdParams: any = {
workspaceId,
linkId: params.linkId,
@@ -180,6 +155,14 @@ export const getAnalytics = async (params: AnalyticsFilters) => {
partnerId: params.partnerId,
tenantId: params.tenantId,
groupId: params.groupId,
+ domain: domainParam,
+ domainOperator,
+ tagIds: tagIdsParam,
+ tagIdsOperator,
+ folderId: folderIdParam,
+ folderIdOperator,
+ root: rootParam,
+ rootOperator,
groupBy,
eventType: event,
start: formatUTCDateTimeClickhouse(startDate),
diff --git a/apps/web/lib/analytics/get-events.ts b/apps/web/lib/analytics/get-events.ts
index 48021fa25b..342e964f88 100644
--- a/apps/web/lib/analytics/get-events.ts
+++ b/apps/web/lib/analytics/get-events.ts
@@ -21,6 +21,7 @@ import {
saleEventResponseSchema,
saleEventSchemaTBEndpoint,
} from "../zod/schemas/sales";
+import { buildAdvancedFilters } from "./build-advanced-filters";
import { queryParser } from "./query-parser";
import { EventsFilters } from "./types";
import { formatUTCDateTimeClickhouse } from "./utils/format-utc-datetime-clickhouse";
@@ -53,14 +54,19 @@ export const getEvents = async (params: EventsFilters) => {
timezone,
});
- if (qr) {
- trigger = "qr";
+ // Handle qr backward compatibility
+ let triggerForPipe = trigger;
+ if (qr && !trigger) {
+ triggerForPipe = { operator: "IS" as const, sqlOperator: "IN" as const, values: ["qr"] };
}
- if (region) {
+ // Handle region split (format: "US-CA")
+ let countryForPipe = country;
+ let regionForPipe = region;
+ if (region && typeof region === 'string') {
const split = region.split("-");
- country = split[0];
- region = split[1];
+ countryForPipe = { operator: "IS" as const, sqlOperator: "IN" as const, values: [split[0]] };
+ regionForPipe = split[1];
}
// support legacy order param
@@ -79,21 +85,72 @@ export const getEvents = async (params: EventsFilters) => {
}[eventType] ?? clickEventSchemaTBEndpoint,
});
- const filters = queryParser(query);
+ const metadataFilters = queryParser(query) || [];
- const response = await pipe({
- ...params,
+ // Build advanced filters for event-level dimensions
+ const advancedFilters = buildAdvancedFilters({
+ country: countryForPipe,
+ trigger: triggerForPipe,
+ city: params.city,
+ continent: params.continent,
+ device: params.device,
+ browser: params.browser,
+ os: params.os,
+ referer: params.referer,
+ refererUrl: params.refererUrl,
+ url: params.url,
+ utm_source: params.utm_source,
+ utm_medium: params.utm_medium,
+ utm_campaign: params.utm_campaign,
+ utm_term: params.utm_term,
+ utm_content: params.utm_content,
+ saleType: params.saleType,
+ });
+
+ const allFilters = [...metadataFilters, ...advancedFilters];
+
+ // Extract workspace_links parameters as direct Tinybird params
+ const domainParam = params.domain?.values;
+ const domainOperator = params.domain?.sqlOperator === 'NOT IN' ? 'NOT IN' : 'IN';
+
+ const tagIdsParam = params.tagIds?.values;
+ const tagIdsOperator = params.tagIds?.sqlOperator === 'NOT IN' ? 'NOT IN' : 'IN';
+
+ const folderIdParam = params.folderId?.values;
+ const folderIdOperator = params.folderId?.sqlOperator === 'NOT IN' ? 'NOT IN' : 'IN';
+
+ const rootParam = params.root?.values;
+ const rootOperator = params.root?.sqlOperator === 'NOT IN' ? 'NOT IN' : 'IN';
+
+ const tinybirdParams: any = {
eventType,
workspaceId,
- trigger,
- country,
- region,
+ linkId: params.linkId,
+ linkIds: params.linkIds,
+ folderIds: params.folderIds,
+ customerId: params.customerId,
+ programId: params.programId,
+ partnerId: params.partnerId,
+ tenantId: params.tenantId,
+ groupId: params.groupId,
+ ...(typeof triggerForPipe !== 'object' && triggerForPipe ? { trigger: triggerForPipe } : {}),
+ ...(typeof countryForPipe !== 'object' && countryForPipe ? { country: countryForPipe } : {}),
+ ...(regionForPipe ? { region: regionForPipe } : {}),
+ // Workspace links filters with operators
+ ...(domainParam ? { domain: domainParam, domainOperator } : {}),
+ ...(tagIdsParam ? { tagIds: tagIdsParam, tagIdsOperator } : {}),
+ ...(folderIdParam ? { folderId: folderIdParam, folderIdOperator } : {}),
+ ...(rootParam ? { root: rootParam, rootOperator } : {}),
order: sortOrder,
offset: (params.page - 1) * params.limit,
+ limit: params.limit,
+ sortBy: params.sortBy,
start: formatUTCDateTimeClickhouse(startDate),
end: formatUTCDateTimeClickhouse(endDate),
- filters: filters ? JSON.stringify(filters) : undefined,
- });
+ filters: allFilters.length > 0 ? JSON.stringify(allFilters) : undefined,
+ };
+
+ const response = await pipe(tinybirdParams);
const [linksMap, customersMap] = await Promise.all([
getLinksMap(response.data.map((d) => d.link_id)),
@@ -118,6 +175,11 @@ export const getEvents = async (params: EventsFilters) => {
link = decodeLinkIfCaseSensitive(link);
+ const transformedLink = transformLink(link, { skipDecodeKey: true });
+ if (transformedLink.testVariants && !Array.isArray(transformedLink.testVariants)) {
+ transformedLink.testVariants = null;
+ }
+
const eventData = {
...evt,
// use link domain & key from mysql instead of tinybird
@@ -133,7 +195,7 @@ export const getEvents = async (params: EventsFilters) => {
refererUrl: evt.referer_url_processed ?? "",
}),
// transformLink -> add shortLink, qrCode, workspaceId, etc.
- link: transformLink(link, { skipDecodeKey: true }),
+ link: transformedLink,
...(evt.event === "lead" || evt.event === "sale"
? {
eventId: evt.event_id,
diff --git a/apps/web/lib/zod/schemas/analytics.ts b/apps/web/lib/zod/schemas/analytics.ts
index 9d11ed74d4..6c236db754 100644
--- a/apps/web/lib/zod/schemas/analytics.ts
+++ b/apps/web/lib/zod/schemas/analytics.ts
@@ -453,6 +453,33 @@ export const analyticsFilterTB = z.object({
.transform((v) => (Array.isArray(v) ? v : v.split(",")))
.optional()
.describe("The folder IDs to retrieve analytics for."),
+ folderId: z
+ .union([z.string(), z.array(z.string())])
+ .transform((v) => (Array.isArray(v) ? v : v.split(",")))
+ .optional()
+ .describe("The folder ID(s) to retrieve analytics for (with operator support)."),
+ domain: z
+ .union([z.string(), z.array(z.string())])
+ .transform((v) => (Array.isArray(v) ? v : v.split(",")))
+ .optional()
+ .describe("The domain(s) to retrieve analytics for."),
+ domainOperator: z.enum(["IN", "NOT IN"]).optional(),
+ tagIds: z
+ .union([z.string(), z.array(z.string())])
+ .transform((v) => (Array.isArray(v) ? v : v.split(",")))
+ .optional()
+ .describe("The tag IDs to retrieve analytics for."),
+ tagIdsOperator: z.enum(["IN", "NOT IN"]).optional(),
+ folderIdOperator: z.enum(["IN", "NOT IN"]).optional(),
+ root: z
+ .union([z.string(), z.boolean(), z.array(z.union([z.string(), z.boolean()]))])
+ .transform((v) => {
+ if (Array.isArray(v)) return v.map(val => typeof val === 'boolean' ? val : val === 'true');
+ return typeof v === 'boolean' ? [v] : [v === 'true'];
+ })
+ .optional()
+ .describe("Filter for root domain links."),
+ rootOperator: z.enum(["IN", "NOT IN"]).optional(),
// Program/Partner/Group filters (not using advanced filtering)
programId: z.string().optional(),
partnerId: z.string().optional(),
diff --git a/apps/web/ui/analytics/toggle.tsx b/apps/web/ui/analytics/toggle.tsx
index 38cff9d622..ddc6856fc3 100644
--- a/apps/web/ui/analytics/toggle.tsx
+++ b/apps/web/ui/analytics/toggle.tsx
@@ -83,6 +83,7 @@ export function AnalyticsToggle({
onSelect={onSelect}
onRemove={onRemove}
onOpenFilter={onOpenFilter}
+ isMultiple={true}
askAI
/>
);
@@ -293,6 +294,7 @@ export function AnalyticsToggle({
onRemoveFilter={onRemoveFilter}
onRemoveAll={onRemoveAll}
onToggleOperator={onToggleOperator}
+ isMultiple={true}
/>
<div
className={cn(
diff --git a/packages/tinybird/pipes/v3_count.pipe b/packages/tinybird/pipes/v3_count.pipe
index af87f72657..cbab70af97 100644
--- a/packages/tinybird/pipes/v3_count.pipe
+++ b/packages/tinybird/pipes/v3_count.pipe
@@ -28,12 +28,33 @@ SQL >
{% if defined(folderIds) %} AND folder_id IN {{ Array(folderIds, 'String') }}
{% elif defined(folderId) %} AND folder_id = {{ folderId }}
{% end %}
- {% if defined(domain) %} AND domain IN {{ Array(domain, 'String') }} {% end %}
+ {% if defined(domain) %}
+ {% if defined(domainOperator) and String(domainOperator) == 'NOT IN' %}
+ AND domain NOT IN {{ Array(domain, 'String') }}
+ {% else %}
+ AND domain IN {{ Array(domain, 'String') }}
+ {% end %}
+ {% end %}
{% if defined(tagIds) %}
- AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != []
+ {% if defined(tagIdsOperator) and String(tagIdsOperator) == 'NOT IN' %}
+ AND length(arrayFilter(x -> x NOT IN {{ Array(tagIds, 'String') }}, tag_ids)) = length(tag_ids)
+ {% else %}
+ AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != []
+ {% end %}
+ {% end %}
+ {% if defined(folderId) %}
+ {% if defined(folderIdOperator) and String(folderIdOperator) == 'NOT IN' %}
+ AND folder_id NOT IN {{ Array(folderId, 'String') }}
+ {% else %}
+ AND folder_id IN {{ Array(folderId, 'String') }}
+ {% end %}
{% end %}
{% if defined(root) %}
- {% if Boolean(root) == 1 %} AND key = '_root' {% else %} AND key != '_root' {% end %}
+ {% if defined(rootOperator) and String(rootOperator) == 'NOT IN' %}
+ AND NOT (0{% for val in root %}{% if val %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %})
+ {% else %}
+ AND (0{% for val in root %}{% if val %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %})
+ {% end %}
{% end %}
diff --git a/packages/tinybird/pipes/v3_events.pipe b/packages/tinybird/pipes/v3_events.pipe
index caf12c54e1..8b73e3c279 100644
--- a/packages/tinybird/pipes/v3_events.pipe
+++ b/packages/tinybird/pipes/v3_events.pipe
@@ -28,12 +28,33 @@ SQL >
{% if defined(folderIds) %} AND folder_id IN {{ Array(folderIds, 'String') }}
{% elif defined(folderId) %} AND folder_id = {{ folderId }}
{% end %}
- {% if defined(domain) %} AND domain IN {{ Array(domain, 'String') }} {% end %}
+ {% if defined(domain) %}
+ {% if defined(domainOperator) and String(domainOperator) == 'NOT IN' %}
+ AND domain NOT IN {{ Array(domain, 'String') }}
+ {% else %}
+ AND domain IN {{ Array(domain, 'String') }}
+ {% end %}
+ {% end %}
{% if defined(tagIds) %}
- AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != []
+ {% if defined(tagIdsOperator) and String(tagIdsOperator) == 'NOT IN' %}
+ AND length(arrayFilter(x -> x NOT IN {{ Array(tagIds, 'String') }}, tag_ids)) = length(tag_ids)
+ {% else %}
+ AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != []
+ {% end %}
+ {% end %}
+ {% if defined(folderId) %}
+ {% if defined(folderIdOperator) and String(folderIdOperator) == 'NOT IN' %}
+ AND folder_id NOT IN {{ Array(folderId, 'String') }}
+ {% else %}
+ AND folder_id IN {{ Array(folderId, 'String') }}
+ {% end %}
{% end %}
{% if defined(root) %}
- {% if Boolean(root) == 1 %} AND key = '_root' {% else %} AND key != '_root' {% end %}
+ {% if defined(rootOperator) and String(rootOperator) == 'NOT IN' %}
+ AND NOT (0{% for val in root %}{% if val %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %})
+ {% else %}
+ AND (0{% for val in root %}{% if val %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %})
+ {% end %}
{% end %}
@@ -82,24 +103,130 @@ SQL >
{% if defined(referer) %} AND referer = {{ referer }} {% end %}
{% if defined(refererUrl) %} AND splitByString('?', referer_url)[1] = {{ refererUrl }} {% end %}
{% if defined(utm_source) %}
- AND url LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%')
+ AND url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%')
{% end %}
{% if defined(utm_medium) %}
- AND url LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%')
+ AND url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%')
{% end %}
{% if defined(utm_campaign) %}
- AND url
- LIKE concat('%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%')
+ AND url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%')
{% end %}
{% if defined(utm_term) %}
- AND url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%')
+ AND url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%')
{% end %}
{% if defined(utm_content) %}
- AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%')
+ AND url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%')
{% end %}
{% if defined(url) %} AND splitByString('?', url)[1] = {{ url }} {% end %}
{% if defined(start) %} AND timestamp >= {{ DateTime64(start) }} {% end %}
{% if defined(end) %} AND timestamp <= {{ DateTime64(end) }} {% end %}
+ {% if defined(filters) %}
+ {% for item in JSON(filters, '[]') %}
+ {% if item.get('operand', '').startswith('metadata.') %}
+ {% set metadataKey = item.get('operand', '').split('.')[1] %}
+ {% set operator = item.get('operator', 'equals') %}
+ {% set value = item.get('value', '') %}
+ {% if operator == 'equals' %}
+ AND JSONExtractString(metadata, {{ metadataKey }}) = {{ value }}
+ {% elif operator == 'notEquals' %}
+ AND JSONExtractString(metadata, {{ metadataKey }}) != {{ value }}
+ {% elif operator == 'greaterThan' %}
+ AND JSONExtractString(metadata, {{ metadataKey }}) > {{ value }}
+ {% elif operator == 'lessThan' %}
+ AND JSONExtractString(metadata, {{ metadataKey }}) < {{ value }}
+ {% elif operator == 'greaterThanOrEqual' %}
+ AND JSONExtractString(metadata, {{ metadataKey }}) >= {{ value }}
+ {% elif operator == 'lessThanOrEqual' %}
+ AND JSONExtractString(metadata, {{ metadataKey }}) <= {{ value }}
+ {% end %}
+ {% elif item.get('field', '') %}
+ {% set field = item.get('field', '') %}
+ {% set operator = item.get('operator', 'IN') %}
+ {% set values = item.get('values', []) %}
+ {% if field == 'country' %}
+ {% if operator == 'IN' %} AND country IN {{ Array(values, 'String') }}
+ {% elif operator == 'NOT IN' %} AND country NOT IN {{ Array(values, 'String') }}
+ {% end %}
+ {% elif field == 'city' %}
+ {% if operator == 'IN' %} AND city IN {{ Array(values, 'String') }}
+ {% elif operator == 'NOT IN' %} AND city NOT IN {{ Array(values, 'String') }}
+ {% end %}
+ {% elif field == 'continent' %}
+ {% if operator == 'IN' %} AND continent IN {{ Array(values, 'String') }}
+ {% elif operator == 'NOT IN' %} AND continent NOT IN {{ Array(values, 'String') }}
+ {% end %}
+ {% elif field == 'device' %}
+ {% if operator == 'IN' %} AND device IN {{ Array(values, 'String') }}
+ {% elif operator == 'NOT IN' %} AND device NOT IN {{ Array(values, 'String') }}
+ {% end %}
+ {% elif field == 'browser' %}
+ {% if operator == 'IN' %} AND browser IN {{ Array(values, 'String') }}
+ {% elif operator == 'NOT IN' %} AND browser NOT IN {{ Array(values, 'String') }}
+ {% end %}
+ {% elif field == 'os' %}
+ {% if operator == 'IN' %} AND os IN {{ Array(values, 'String') }}
+ {% elif operator == 'NOT IN' %} AND os NOT IN {{ Array(values, 'String') }}
+ {% end %}
+ {% elif field == 'trigger' %}
+ {% if operator == 'IN' %} AND trigger IN {{ Array(values, 'String') }}
+ {% elif operator == 'NOT IN' %} AND trigger NOT IN {{ Array(values, 'String') }}
+ {% end %}
+ {% elif field == 'referer' %}
+ {% if operator == 'IN' %} AND referer IN {{ Array(values, 'String') }}
+ {% elif operator == 'NOT IN' %} AND referer NOT IN {{ Array(values, 'String') }}
+ {% end %}
+ {% elif field == 'refererUrl' %}
+ {% if operator == 'IN' %} AND splitByString('?', referer_url)[1] IN {{ Array(values, 'String') }}
+ {% elif operator == 'NOT IN' %} AND splitByString('?', referer_url)[1] NOT IN {{ Array(values, 'String') }}
+ {% end %}
+ {% elif field == 'url' %}
+ {% if operator == 'IN' %} AND splitByString('?', url)[1] IN {{ Array(values, 'String') }}
+ {% elif operator == 'NOT IN' %} AND splitByString('?', url)[1] NOT IN {{ Array(values, 'String') }}
+ {% end %}
+ {% elif field == 'utm_source' %}
+ {% if operator == 'IN' %}
+ AND (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% elif operator == 'NOT IN' %}
+ AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% end %}
+ {% elif field == 'utm_medium' %}
+ {% if operator == 'IN' %}
+ AND (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% elif operator == 'NOT IN' %}
+ AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% end %}
+ {% elif field == 'utm_campaign' %}
+ {% if operator == 'IN' %}
+ AND (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% elif operator == 'NOT IN' %}
+ AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% end %}
+ {% elif field == 'utm_term' %}
+ {% if operator == 'IN' %}
+ AND (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% elif operator == 'NOT IN' %}
+ AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% end %}
+ {% elif field == 'utm_content' %}
+ {% if operator == 'IN' %}
+ AND (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% elif operator == 'NOT IN' %}
+ AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% end %}
+ {% elif field == 'domain' %}
+ {# domain is already filtered at workspace_links node level #}
+ {% elif field == 'tagIds' %}
+ {# tagIds is already filtered at workspace_links node level #}
+ {% elif field == 'folderId' %}
+ {# folderId is already filtered at workspace_links node level #}
+ {% elif field == 'root' %}
+ {# root is already filtered at workspace_links node level #}
+ {% elif field == 'saleType' %}
+ {# saleType only applies to sales events #}
+ {% end %}
+ {% end %}
+ {% end %}
+ {% end %}
ORDER BY timestamp {% if order == 'asc' %} ASC {% else %} DESC {% end %}
LIMIT {{ Int32(limit, 100) }}
{% if defined(offset) %} OFFSET {{ Int32(offset, 0) }} {% end %}
@@ -140,20 +267,19 @@ SQL >
{% if defined(referer) %} AND referer = {{ referer }} {% end %}
{% if defined(refererUrl) %} AND splitByString('?', referer_url)[1] = {{ refererUrl }} {% end %}
{% if defined(utm_source) %}
- AND url LIKE concat('%utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%')
+ AND url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(utm_source) }}), '%')
{% end %}
{% if defined(utm_medium) %}
- AND url LIKE concat('%utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%')
+ AND url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(utm_medium) }}), '%')
{% end %}
{% if defined(utm_campaign) %}
- AND url
- LIKE concat('%utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%')
+ AND url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(utm_campaign) }}), '%')
{% end %}
{% if defined(utm_term) %}
- AND url LIKE concat('%utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%')
+ AND url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(utm_term) }}), '%')
{% end %}
{% if defined(utm_content) %}
- AND url LIKE concat('%utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%')
+ AND url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(utm_content) }}), '%')
{% end %}
{% if defined(url) %} AND splitByString('?', url)[1] = {{ url }} {% end %}
{% if defined(start) %} AND timestamp >= {{ DateTime(start) }} {% end %}
@@ -177,6 +303,87 @@ SQL >
{% elif operator == 'lessThanOrEqual' %}
AND JSONExtractString(metadata, {{ metadataKey }}) <= {{ value }}
{% end %}
+ {% elif item.get('field', '') %}
+ {% set field = item.get('field', '') %}
+ {% set operator = item.get('operator', 'IN') %}
+ {% set values = item.get('values', []) %}
+ {% if field == 'country' %}
+ {% if operator == 'IN' %} AND country IN {{ Array(values, 'String') }}
+ {% elif operator == 'NOT IN' %} AND country NOT IN {{ Array(values, 'String') }}
+ {% end %}
+ {% elif field == 'city' %}
+ {% if operator == 'IN' %} AND city IN {{ Array(values, 'String') }}
+ {% elif operator == 'NOT IN' %} AND city NOT IN {{ Array(values, 'String') }}
+ {% end %}
+ {% elif field == 'continent' %}
+ {% if operator == 'IN' %} AND continent IN {{ Array(values, 'String') }}
+ {% elif operator == 'NOT IN' %} AND continent NOT IN {{ Array(values, 'String') }}
+ {% end %}
+ {% elif field == 'device' %}
+ {% if operator == 'IN' %} AND device IN {{ Array(values, 'String') }}
+ {% elif operator == 'NOT IN' %} AND device NOT IN {{ Array(values, 'String') }}
+ {% end %}
+ {% elif field == 'browser' %}
+ {% if operator == 'IN' %} AND browser IN {{ Array(values, 'String') }}
+ {% elif operator == 'NOT IN' %} AND browser NOT IN {{ Array(values, 'String') }}
+ {% end %}
+ {% elif field == 'os' %}
+ {% if operator == 'IN' %} AND os IN {{ Array(values, 'String') }}
+ {% elif operator == 'NOT IN' %} AND os NOT IN {{ Array(values, 'String') }}
+ {% end %}
+ {% elif field == 'referer' %}
+ {% if operator == 'IN' %} AND referer IN {{ Array(values, 'String') }}
+ {% elif operator == 'NOT IN' %} AND referer NOT IN {{ Array(values, 'String') }}
+ {% end %}
+ {% elif field == 'refererUrl' %}
+ {% if operator == 'IN' %} AND splitByString('?', referer_url)[1] IN {{ Array(values, 'String') }}
+ {% elif operator == 'NOT IN' %} AND splitByString('?', referer_url)[1] NOT IN {{ Array(values, 'String') }}
+ {% end %}
+ {% elif field == 'url' %}
+ {% if operator == 'IN' %} AND splitByString('?', url)[1] IN {{ Array(values, 'String') }}
+ {% elif operator == 'NOT IN' %} AND splitByString('?', url)[1] NOT IN {{ Array(values, 'String') }}
+ {% end %}
+ {% elif field == 'utm_source' %}
+ {% if operator == 'IN' %}
+ AND (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% elif operator == 'NOT IN' %}
+ AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_source=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% end %}
+ {% elif field == 'utm_medium' %}
+ {% if operator == 'IN' %}
+ AND (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% elif operator == 'NOT IN' %}
+ AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_medium=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% end %}
+ {% elif field == 'utm_campaign' %}
+ {% if operator == 'IN' %}
+ AND (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% elif operator == 'NOT IN' %}
+ AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_campaign=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% end %}
+ {% elif field == 'utm_term' %}
+ {% if operator == 'IN' %}
+ AND (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% elif operator == 'NOT IN' %}
+ AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_term=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% end %}
+ {% elif field == 'utm_content' %}
+ {% if operator == 'IN' %}
+ AND (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% elif operator == 'NOT IN' %}
+ AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
+ {% end %}
+ {% elif field == 'domain' %}
+ {# domain is already filtered at workspace_links node level #}
+ {% elif field == 'tagIds' %}
+ {# tagIds is already filtered at workspace_links node level #}
+ {% elif field == 'folderId' %}
+ {# folderId is already filtered at workspace_links node level #}
+ {% elif field == 'root' %}
+ {# root is already filtered at workspace_links node level #}
+ {% elif field == 'saleType' %}
+ {# saleType only applies to sales events #}
+ {% end %}
{% end %}
{% end %}
{% end %}
diff --git a/packages/tinybird/pipes/v3_group_by.pipe b/packages/tinybird/pipes/v3_group_by.pipe
index 4863a92c2e..00a8f8a0a1 100644
--- a/packages/tinybird/pipes/v3_group_by.pipe
+++ b/packages/tinybird/pipes/v3_group_by.pipe
@@ -28,12 +28,33 @@ SQL >
{% if defined(folderIds) %} AND folder_id IN {{ Array(folderIds, 'String') }}
{% elif defined(folderId) %} AND folder_id = {{ folderId }}
{% end %}
- {% if defined(domain) %} AND domain IN {{ Array(domain, 'String') }} {% end %}
+ {% if defined(domain) %}
+ {% if defined(domainOperator) and String(domainOperator) == 'NOT IN' %}
+ AND domain NOT IN {{ Array(domain, 'String') }}
+ {% else %}
+ AND domain IN {{ Array(domain, 'String') }}
+ {% end %}
+ {% end %}
{% if defined(tagIds) %}
- AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != []
+ {% if defined(tagIdsOperator) and String(tagIdsOperator) == 'NOT IN' %}
+ AND length(arrayFilter(x -> x NOT IN {{ Array(tagIds, 'String') }}, tag_ids)) = length(tag_ids)
+ {% else %}
+ AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != []
+ {% end %}
+ {% end %}
+ {% if defined(folderId) %}
+ {% if defined(folderIdOperator) and String(folderIdOperator) == 'NOT IN' %}
+ AND folder_id NOT IN {{ Array(folderId, 'String') }}
+ {% else %}
+ AND folder_id IN {{ Array(folderId, 'String') }}
+ {% end %}
{% end %}
{% if defined(root) %}
- {% if Boolean(root) == 1 %} AND key = '_root' {% else %} AND key != '_root' {% end %}
+ {% if defined(rootOperator) and String(rootOperator) == 'NOT IN' %}
+ AND NOT (0{% for val in root %}{% if val %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %})
+ {% else %}
+ AND (0{% for val in root %}{% if val %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %})
+ {% end %}
{% end %}
diff --git a/packages/tinybird/pipes/v3_group_by_link_metadata.pipe b/packages/tinybird/pipes/v3_group_by_link_metadata.pipe
index aeae867436..8fd2751cbd 100644
--- a/packages/tinybird/pipes/v3_group_by_link_metadata.pipe
+++ b/packages/tinybird/pipes/v3_group_by_link_metadata.pipe
@@ -49,15 +49,36 @@ SQL >
{% if defined(partnerId) %} AND partner_id = {{ partnerId }} {% end %}
{% if defined(groupId) %} AND partner_group_id = {{ groupId }} {% end %}
{% if defined(tenantId) %} AND tenant_id = {{ tenantId }} {% end %}
- {% if defined(domain) %} AND domain IN {{ Array(domain, 'String') }} {% end %}
{% if defined(folderIds) %} AND folder_id IN {{ Array(folderIds, 'String') }}
{% elif defined(folderId) %} AND folder_id = {{ folderId }}
{% end %}
+ {% if defined(domain) %}
+ {% if defined(domainOperator) and String(domainOperator) == 'NOT IN' %}
+ AND domain NOT IN {{ Array(domain, 'String') }}
+ {% else %}
+ AND domain IN {{ Array(domain, 'String') }}
+ {% end %}
+ {% end %}
{% if defined(tagIds) %}
- AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != []
+ {% if defined(tagIdsOperator) and String(tagIdsOperator) == 'NOT IN' %}
+ AND length(arrayFilter(x -> x NOT IN {{ Array(tagIds, 'String') }}, tag_ids)) = length(tag_ids)
+ {% else %}
+ AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != []
+ {% end %}
+ {% end %}
+ {% if defined(folderId) %}
+ {% if defined(folderIdOperator) and String(folderIdOperator) == 'NOT IN' %}
+ AND folder_id NOT IN {{ Array(folderId, 'String') }}
+ {% else %}
+ AND folder_id IN {{ Array(folderId, 'String') }}
+ {% end %}
{% end %}
{% if defined(root) %}
- {% if Boolean(root) == 1 %} AND key = '_root' {% else %} AND key != '_root' {% end %}
+ {% if defined(rootOperator) and String(rootOperator) == 'NOT IN' %}
+ AND NOT (0{% for val in root %}{% if val %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %})
+ {% else %}
+ AND (0{% for val in root %}{% if val %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %})
+ {% end %}
{% end %}
@@ -162,23 +183,13 @@ SQL >
AND NOT (0{% for val in values %} OR url LIKE concat('%', 'utm_content=', encodeURLFormComponent({{ String(val) }}), '%'){% end %})
{% end %}
{% elif field == 'domain' %}
- {% if operator == 'IN' %} AND ce.link_id IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }})
- {% elif operator == 'NOT IN' %} AND ce.link_id NOT IN (SELECT link_id FROM workspace_links WHERE domain IN {{ Array(values, 'String') }})
- {% end %}
+ {# domain is already filtered at workspace_links node level #}
{% elif field == 'tagIds' %}
- {% if operator == 'IN' %} AND ce.link_id IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != [])
- {% elif operator == 'NOT IN' %} AND ce.link_id NOT IN (SELECT link_id FROM workspace_links WHERE arrayIntersect(tag_ids, {{ Array(values, 'String') }}) != [])
- {% end %}
+ {# tagIds is already filtered at workspace_links node level #}
{% elif field == 'folderId' %}
- {% if operator == 'IN' %} AND ce.link_id IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }})
- {% elif operator == 'NOT IN' %} AND ce.link_id NOT IN (SELECT link_id FROM workspace_links WHERE folder_id IN {{ Array(values, 'String') }})
- {% end %}
+ {# folderId is already filtered at workspace_links node level #}
{% elif field == 'root' %}
- {% if operator == 'IN' %}
- AND ce.link_id IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %})
- {% elif operator == 'NOT IN' %}
- AND ce.link_id NOT IN (SELECT link_id FROM workspace_links WHERE 0{% for val in values %} OR {% if val == 'true' %}key = '_root'{% else %}key != '_root'{% end %}{% end %})
- {% end %}
+ {# root is already filtered at workspace_links node level #}
{% elif field == 'saleType' %}
{# saleType is handled in the sales subquery, not here #}
{% end %}
diff --git a/packages/tinybird/pipes/v3_timeseries.pipe b/packages/tinybird/pipes/v3_timeseries.pipe
index 7030bb7904..bfc0548fd1 100644
--- a/packages/tinybird/pipes/v3_timeseries.pipe
+++ b/packages/tinybird/pipes/v3_timeseries.pipe
@@ -101,12 +101,33 @@ SQL >
{% if defined(folderIds) %} AND folder_id IN {{ Array(folderIds, 'String') }}
{% elif defined(folderId) %} AND folder_id = {{ folderId }}
{% end %}
- {% if defined(domain) %} AND domain IN {{ Array(domain, 'String') }} {% end %}
+ {% if defined(domain) %}
+ {% if defined(domainOperator) and String(domainOperator) == 'NOT IN' %}
+ AND domain NOT IN {{ Array(domain, 'String') }}
+ {% else %}
+ AND domain IN {{ Array(domain, 'String') }}
+ {% end %}
+ {% end %}
{% if defined(tagIds) %}
- AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != []
+ {% if defined(tagIdsOperator) and String(tagIdsOperator) == 'NOT IN' %}
+ AND length(arrayFilter(x -> x NOT IN {{ Array(tagIds, 'String') }}, tag_ids)) = length(tag_ids)
+ {% else %}
+ AND arrayIntersect(tag_ids, {{ Array(tagIds, 'String') }}) != []
+ {% end %}
+ {% end %}
+ {% if defined(folderId) %}
+ {% if defined(folderIdOperator) and String(folderIdOperator) == 'NOT IN' %}
+ AND folder_id NOT IN {{ Array(folderId, 'String') }}
+ {% else %}
+ AND folder_id IN {{ Array(folderId, 'String') }}
+ {% end %}
{% end %}
{% if defined(root) %}
- {% if Boolean(root) == 1 %} AND key = '_root' {% else %} AND key != '_root' {% end %}
+ {% if defined(rootOperator) and String(rootOperator) == 'NOT IN' %}
+ AND NOT (0{% for val in root %}{% if val %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %})
+ {% else %}
+ AND (0{% for val in root %}{% if val %} OR key = '_root'{% else %} OR key != '_root'{% end %}{% end %})
+ {% end %}
{% end %}