Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Remove ARRAY JOINs (#3708)
Karl-Aksel Puulmann11c5d3b251a0
diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex b5e1cae998..e9208088e3 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -1,6 +1,8 @@defmodule Plausible.Stats.Base douse Plausible.ClickhouseRepouse Plausible+ use Plausible.Stats.Fragments+alias Plausible.Stats.{Query, Filters}alias Plausible.Timezonesimport Ecto.Query@@ -114,14 +116,12 @@ defmodule Plausible.Stats.Base doif value == "(none)" dofrom(e in q,- where: fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name)+ where: not has_key(e, :meta, ^prop_name))elsefrom(e in q,- array_join: meta in "meta",- as: :meta,- where: meta.key == ^prop_name and meta.value == ^value+ where: has_key(e, :meta, ^prop_name) and get_by_key(e, :meta, ^prop_name) == ^value)end@@ -129,16 +129,13 @@ defmodule Plausible.Stats.Base doif value == "(none)" dofrom(e in q,- where: fragment("has(?, ?)", field(e, :"meta.key"), ^prop_name)+ where: has_key(e, :meta, ^prop_name))elsefrom(e in q,- left_array_join: meta in "meta",- as: :meta,where:- (meta.key == ^prop_name and meta.value != ^value) or- fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name)+ not has_key(e, :meta, ^prop_name) or get_by_key(e, :meta, ^prop_name) != ^value)end@@ -147,9 +144,9 @@ defmodule Plausible.Stats.Base dofrom(e in q,- left_array_join: meta in "meta",- as: :meta,- where: meta.key == ^prop_name and fragment("match(?, ?)", meta.value, ^regex)+ where:+ has_key(e, :meta, ^prop_name) and+ fragment("match(?, ?)", get_by_key(e, :meta, ^prop_name), ^regex)){"event:props:" <> prop_name, {:member, values}} ->@@ -157,12 +154,9 @@ defmodule Plausible.Stats.Base dofrom(e in q,- left_array_join: meta in "meta",- as: :meta,where:- (meta.key == ^prop_name and meta.value in ^values) or- (^none_value_included and- fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name))+ (has_key(e, :meta, ^prop_name) and get_by_key(e, :meta, ^prop_name) in ^values) or+ (^none_value_included and not has_key(e, :meta, ^prop_name))){"event:props:" <> prop_name, {:not_member, values}} ->@@ -170,14 +164,13 @@ defmodule Plausible.Stats.Base dofrom(e in q,- left_array_join: meta in "meta",- as: :meta,where:- (meta.key == ^prop_name and meta.value not in ^values) or- (^none_value_included and fragment("has(?, ?)", field(e, :"meta.key"), ^prop_name) and- meta.value not in ^values) or- (not (^none_value_included) and- fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name))+ (has_key(e, :meta, ^prop_name) and+ get_by_key(e, :meta, ^prop_name) not in ^values) or+ (^none_value_included and+ has_key(e, :meta, ^prop_name) and+ get_by_key(e, :meta, ^prop_name) not in ^values) or+ (not (^none_value_included) and not has_key(e, :meta, ^prop_name)))_ ->@@ -240,34 +233,31 @@ defmodule Plausible.Stats.Base dodef apply_entry_prop_filter(sessions_q, prop_name, {:is, "(none)"}) dofrom(s in sessions_q,- where: fragment("not has(?, ?)", field(s, :"entry_meta.key"), ^prop_name)+ where: not has_key(s, :entry_meta, ^prop_name))enddef apply_entry_prop_filter(sessions_q, prop_name, {:is, value}) dofrom(s in sessions_q,- array_join: meta in "entry_meta",- as: :meta,- where: meta.key == ^prop_name and meta.value == ^value+ where:+ has_key(s, :entry_meta, ^prop_name) and get_by_key(s, :entry_meta, ^prop_name) == ^value)enddef apply_entry_prop_filter(sessions_q, prop_name, {:is_not, "(none)"}) dofrom(s in sessions_q,- where: fragment("has(?, ?)", field(s, :"entry_meta.key"), ^prop_name)+ where: has_key(s, :entry_meta, ^prop_name))enddef apply_entry_prop_filter(sessions_q, prop_name, {:is_not, value}) dofrom(s in sessions_q,- left_array_join: meta in "entry_meta",- as: :meta,where:- (meta.key == ^prop_name and meta.value != ^value) or- fragment("not has(?, ?)", field(s, :"entry_meta.key"), ^prop_name)+ not has_key(s, :entry_meta, ^prop_name) or+ get_by_key(s, :entry_meta, ^prop_name) != ^value)enddiff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 0d5377485e..cf3e93d3c3 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -1,6 +1,8 @@defmodule Plausible.Stats.Breakdown douse Plausible.ClickhouseRepouse Plausible+ use Plausible.Stats.Fragments+import Plausible.Stats.{Base, Imported, Util}require OpenTelemetry.Tracer, as: Traceralias Plausible.Stats.Query@@ -397,37 +399,16 @@ defmodule Plausible.Stats.Breakdown do|> Map.new()end- defp joins_table?(ecto_q, table) do- Enum.any?(- ecto_q.joins,- fn- %Ecto.Query.JoinExpr{source: {^table, _}} -> true- _ -> false- end- )- end-defp do_group_by(%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,"event:props:" <> prop) do- q =- if joins_table?(q, "meta") do- q- else- from(- e in q,- array_join: meta in fragment("meta"),- as: :meta- )- end-from(- [e, meta: meta] in q,- where: meta.key == ^prop,- group_by: meta.value,- select_merge: %{^prop => meta.value},- order_by: {:asc, meta.value}+ e in q,+ where: has_key(e, :meta, ^prop),+ select_merge: %{^prop => get_by_key(e, :meta, ^prop)},+ group_by: get_by_key(e, :meta, ^prop),+ order_by: {:asc, get_by_key(e, :meta, ^prop)})enddiff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex a3e2f9a8fa..ce116f491b 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -538,13 +538,12 @@ defmodule Plausible.Stats.Clickhouse doif val == "(none)" dofrom(e in q,- where: fragment("not has(meta.key, ?)", ^key)+ where: not has_key(e, :meta, ^key))elsefrom(e in q,- array_join: meta in fragment("meta"),- where: meta.key == ^key and meta.value == ^val+ where: has_key(e, :meta, ^key) and get_by_key(e, :meta, ^key) == ^val)endelsediff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.exindex 0e5d979867..ae71f2de13 100644--- a/lib/plausible/stats/filter_suggestions.ex+++ b/lib/plausible/stats/filter_suggestions.ex@@ -1,6 +1,7 @@defmodule Plausible.Stats.FilterSuggestions douse Plausible.Repouse Plausible.ClickhouseRepo+ use Plausible.Stats.Fragmentsimport Plausible.Stats.Basealias Plausible.Stats.Query@@ -154,20 +155,22 @@ defmodule Plausible.Stats.FilterSuggestions donone_q =from(e in base_event_query(site, Query.remove_event_filters(query, [:props])),- left_array_join: meta in "meta",- as: :meta,select: "(none)",- where: fragment("not has(?, ?)", field(e, :"meta.key"), ^key),+ where: not has_key(e, :meta, ^key),limit: 1)search_q =from(e in base_event_query(site, query),- array_join: meta in "meta",- as: :meta,- select: meta.value,- where: meta.key == ^key and fragment("? ilike ?", meta.value, ^filter_query),- group_by: meta.value,+ select: get_by_key(e, :meta, ^key),+ where:+ has_key(e, :meta, ^key) and+ fragment(+ "? ilike ?",+ get_by_key(e, :meta, ^key),+ ^filter_query+ ),+ group_by: get_by_key(e, :meta, ^key),order_by: [desc: fragment("count(*)")],limit: 25)diff --git a/lib/plausible/stats/fragments.ex b/lib/plausible/stats/fragments.exindex a4caea33b5..ae5ef4e44c 100644--- a/lib/plausible/stats/fragments.ex+++ b/lib/plausible/stats/fragments.ex@@ -97,6 +97,52 @@ defmodule Plausible.Stats.Fragments doendend+ @doc """+ Returns whether a key (usually property) exists under `meta.key` array or similar.++ This macro is used for operating on custom properties.++ ## Examples++ `has_key(e, :meta, "some_property_name")` expands to SQL `has(meta.key, "some_property_name")`+ """+ defmacro has_key(table, meta_column, key) do+ quote do+ fragment(+ "has(?, ?)",+ field(unquote(table), unquote(meta_key_column(meta_column))),+ unquote(key)+ )+ end+ end++ @doc """+ Returns value of a key (usually property) under `meta.value` array or similar.++ This macro is used for operating on custom properties.+ Callsites should also check whether key exists first in SQL via `has_key` macro.++ ## Examples++ `get_by_key(e, :meta, "some_property_name")` expands to SQL `meta.value[indexOf(meta.key, "some_property")]`+ """+ defmacro get_by_key(table, meta_column, key) do+ quote do+ fragment(+ "?[indexOf(?, ?)]",+ field(unquote(table), unquote(meta_value_column(meta_column))),+ field(unquote(table), unquote(meta_key_column(meta_column))),+ unquote(key)+ )+ end+ end++ defp meta_key_column(:meta), do: :"meta.key"+ defp meta_key_column(:entry_meta), do: :"entry_meta.key"++ defp meta_value_column(:meta), do: :"meta.value"+ defp meta_value_column(:entry_meta), do: :"entry_meta.value"+defmacro __using__(_) doquote doimport Plausible.Stats.Fragments