Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Remove ARRAY JOINs (#3708)

Karl-Aksel Puulmann11c5d3b251a0
diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex
index b5e1cae998..e9208088e3 100644
--- a/lib/plausible/stats/base.ex
+++ b/lib/plausible/stats/base.ex
@@ -1,6 +1,8 @@
defmodule Plausible.Stats.Base do
use Plausible.ClickhouseRepo
use Plausible
+ use Plausible.Stats.Fragments
+
alias Plausible.Stats.{Query, Filters}
alias Plausible.Timezones
import Ecto.Query
@@ -114,14 +116,12 @@ defmodule Plausible.Stats.Base do
if value == "(none)" do
from(
e in q,
- where: fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name)
+ where: not has_key(e, :meta, ^prop_name)
)
else
from(
e in q,
- array_join: meta in "meta",
- as: :meta,
- where: meta.key == ^prop_name and meta.value == ^value
+ where: has_key(e, :meta, ^prop_name) and get_by_key(e, :meta, ^prop_name) == ^value
)
end
@@ -129,16 +129,13 @@ defmodule Plausible.Stats.Base do
if value == "(none)" do
from(
e in q,
- where: fragment("has(?, ?)", field(e, :"meta.key"), ^prop_name)
+ where: has_key(e, :meta, ^prop_name)
)
else
from(
e in q,
- left_array_join: meta in "meta",
- as: :meta,
where:
- (meta.key == ^prop_name and meta.value != ^value) or
- fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name)
+ not has_key(e, :meta, ^prop_name) or get_by_key(e, :meta, ^prop_name) != ^value
)
end
@@ -147,9 +144,9 @@ defmodule Plausible.Stats.Base do
from(
e in q,
- left_array_join: meta in "meta",
- as: :meta,
- where: meta.key == ^prop_name and fragment("match(?, ?)", meta.value, ^regex)
+ where:
+ has_key(e, :meta, ^prop_name) and
+ fragment("match(?, ?)", get_by_key(e, :meta, ^prop_name), ^regex)
)
{"event:props:" <> prop_name, {:member, values}} ->
@@ -157,12 +154,9 @@ defmodule Plausible.Stats.Base do
from(
e in q,
- left_array_join: meta in "meta",
- as: :meta,
where:
- (meta.key == ^prop_name and meta.value in ^values) or
- (^none_value_included and
- fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name))
+ (has_key(e, :meta, ^prop_name) and get_by_key(e, :meta, ^prop_name) in ^values) or
+ (^none_value_included and not has_key(e, :meta, ^prop_name))
)
{"event:props:" <> prop_name, {:not_member, values}} ->
@@ -170,14 +164,13 @@ defmodule Plausible.Stats.Base do
from(
e in q,
- left_array_join: meta in "meta",
- as: :meta,
where:
- (meta.key == ^prop_name and meta.value not in ^values) or
- (^none_value_included and fragment("has(?, ?)", field(e, :"meta.key"), ^prop_name) and
- meta.value not in ^values) or
- (not (^none_value_included) and
- fragment("not has(?, ?)", field(e, :"meta.key"), ^prop_name))
+ (has_key(e, :meta, ^prop_name) and
+ get_by_key(e, :meta, ^prop_name) not in ^values) or
+ (^none_value_included and
+ has_key(e, :meta, ^prop_name) and
+ get_by_key(e, :meta, ^prop_name) not in ^values) or
+ (not (^none_value_included) and not has_key(e, :meta, ^prop_name))
)
_ ->
@@ -240,34 +233,31 @@ defmodule Plausible.Stats.Base do
def apply_entry_prop_filter(sessions_q, prop_name, {:is, "(none)"}) do
from(
s in sessions_q,
- where: fragment("not has(?, ?)", field(s, :"entry_meta.key"), ^prop_name)
+ where: not has_key(s, :entry_meta, ^prop_name)
)
end
def apply_entry_prop_filter(sessions_q, prop_name, {:is, value}) do
from(
s in sessions_q,
- array_join: meta in "entry_meta",
- as: :meta,
- where: meta.key == ^prop_name and meta.value == ^value
+ where:
+ has_key(s, :entry_meta, ^prop_name) and get_by_key(s, :entry_meta, ^prop_name) == ^value
)
end
def apply_entry_prop_filter(sessions_q, prop_name, {:is_not, "(none)"}) do
from(
s in sessions_q,
- where: fragment("has(?, ?)", field(s, :"entry_meta.key"), ^prop_name)
+ where: has_key(s, :entry_meta, ^prop_name)
)
end
def apply_entry_prop_filter(sessions_q, prop_name, {:is_not, value}) do
from(
s in sessions_q,
- left_array_join: meta in "entry_meta",
- as: :meta,
where:
- (meta.key == ^prop_name and meta.value != ^value) or
- fragment("not has(?, ?)", field(s, :"entry_meta.key"), ^prop_name)
+ not has_key(s, :entry_meta, ^prop_name) or
+ get_by_key(s, :entry_meta, ^prop_name) != ^value
)
end
diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index 0d5377485e..cf3e93d3c3 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -1,6 +1,8 @@
defmodule Plausible.Stats.Breakdown do
use Plausible.ClickhouseRepo
use Plausible
+ use Plausible.Stats.Fragments
+
import Plausible.Stats.{Base, Imported, Util}
require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.Query
@@ -397,37 +399,16 @@ defmodule Plausible.Stats.Breakdown do
|> Map.new()
end
- defp joins_table?(ecto_q, table) do
- Enum.any?(
- ecto_q.joins,
- fn
- %Ecto.Query.JoinExpr{source: {^table, _}} -> true
- _ -> false
- end
- )
- end
-
defp do_group_by(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,
"event:props:" <> prop
) do
- q =
- if joins_table?(q, "meta") do
- q
- else
- from(
- e in q,
- array_join: meta in fragment("meta"),
- as: :meta
- )
- end
-
from(
- [e, meta: meta] in q,
- where: meta.key == ^prop,
- group_by: meta.value,
- select_merge: %{^prop => meta.value},
- order_by: {:asc, meta.value}
+ e in q,
+ where: has_key(e, :meta, ^prop),
+ select_merge: %{^prop => get_by_key(e, :meta, ^prop)},
+ group_by: get_by_key(e, :meta, ^prop),
+ order_by: {:asc, get_by_key(e, :meta, ^prop)}
)
end
diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex
index a3e2f9a8fa..ce116f491b 100644
--- a/lib/plausible/stats/clickhouse.ex
+++ b/lib/plausible/stats/clickhouse.ex
@@ -538,13 +538,12 @@ defmodule Plausible.Stats.Clickhouse do
if val == "(none)" do
from(
e in q,
- where: fragment("not has(meta.key, ?)", ^key)
+ where: not has_key(e, :meta, ^key)
)
else
from(
e in q,
- array_join: meta in fragment("meta"),
- where: meta.key == ^key and meta.value == ^val
+ where: has_key(e, :meta, ^key) and get_by_key(e, :meta, ^key) == ^val
)
end
else
diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.ex
index 0e5d979867..ae71f2de13 100644
--- a/lib/plausible/stats/filter_suggestions.ex
+++ b/lib/plausible/stats/filter_suggestions.ex
@@ -1,6 +1,7 @@
defmodule Plausible.Stats.FilterSuggestions do
use Plausible.Repo
use Plausible.ClickhouseRepo
+ use Plausible.Stats.Fragments
import Plausible.Stats.Base
alias Plausible.Stats.Query
@@ -154,20 +155,22 @@ defmodule Plausible.Stats.FilterSuggestions do
none_q =
from(e in base_event_query(site, Query.remove_event_filters(query, [:props])),
- left_array_join: meta in "meta",
- as: :meta,
select: "(none)",
- where: fragment("not has(?, ?)", field(e, :"meta.key"), ^key),
+ where: not has_key(e, :meta, ^key),
limit: 1
)
search_q =
from(e in base_event_query(site, query),
- array_join: meta in "meta",
- as: :meta,
- select: meta.value,
- where: meta.key == ^key and fragment("? ilike ?", meta.value, ^filter_query),
- group_by: meta.value,
+ select: get_by_key(e, :meta, ^key),
+ where:
+ has_key(e, :meta, ^key) and
+ fragment(
+ "? ilike ?",
+ get_by_key(e, :meta, ^key),
+ ^filter_query
+ ),
+ group_by: get_by_key(e, :meta, ^key),
order_by: [desc: fragment("count(*)")],
limit: 25
)
diff --git a/lib/plausible/stats/fragments.ex b/lib/plausible/stats/fragments.ex
index a4caea33b5..ae5ef4e44c 100644
--- a/lib/plausible/stats/fragments.ex
+++ b/lib/plausible/stats/fragments.ex
@@ -97,6 +97,52 @@ defmodule Plausible.Stats.Fragments do
end
end
+ @doc """
+ Returns whether a key (usually property) exists under `meta.key` array or similar.
+
+ This macro is used for operating on custom properties.
+
+ ## Examples
+
+ `has_key(e, :meta, "some_property_name")` expands to SQL `has(meta.key, "some_property_name")`
+ """
+ defmacro has_key(table, meta_column, key) do
+ quote do
+ fragment(
+ "has(?, ?)",
+ field(unquote(table), unquote(meta_key_column(meta_column))),
+ unquote(key)
+ )
+ end
+ end
+
+ @doc """
+ Returns value of a key (usually property) under `meta.value` array or similar.
+
+ This macro is used for operating on custom properties.
+ Callsites should also check whether key exists first in SQL via `has_key` macro.
+
+ ## Examples
+
+ `get_by_key(e, :meta, "some_property_name")` expands to SQL `meta.value[indexOf(meta.key, "some_property")]`
+ """
+ defmacro get_by_key(table, meta_column, key) do
+ quote do
+ fragment(
+ "?[indexOf(?, ?)]",
+ field(unquote(table), unquote(meta_value_column(meta_column))),
+ field(unquote(table), unquote(meta_key_column(meta_column))),
+ unquote(key)
+ )
+ end
+ end
+
+ defp meta_key_column(:meta), do: :"meta.key"
+ defp meta_key_column(:entry_meta), do: :"entry_meta.key"
+
+ defp meta_value_column(:meta), do: :"meta.value"
+ defp meta_value_column(:entry_meta), do: :"entry_meta.value"
+
defmacro __using__(_) do
quote do
import Plausible.Stats.Fragments