Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Data Migration: rewrite funnels that contain duplicate goals (#3361)

hq1542385feabbb
diff --git a/lib/plausible/data_migration/rewrite_funnel_dupes.ex b/lib/plausible/data_migration/rewrite_funnel_dupes.ex
new file mode 100644
index 0000000000..52db56d347
--- /dev/null
+++ b/lib/plausible/data_migration/rewrite_funnel_dupes.ex
@@ -0,0 +1,191 @@
+defmodule Plausible.DataMigration.RewriteFunnelDupes do
+ @moduledoc """
+ A data fix migration that seeks funnels having steps
+ whose goals are equally named.
+ It then tries to rewrite the duplicate goals using the
+ oldest goal available. In extreme cases, e.g. when multiple
+ duplicates are found for a single funnel, it will either
+ reduce or completely remove the funnel.
+ This enables us to run a migration later on that will
+ delete duplicate goals and enforce goal uniqueness at the
+ database level via a CHECK constraint.
+
+ To run, just call the `run` function.
+ """
+ use Plausible.DataMigration, dir: "FunnelDupeGoals", repo: Plausible.Repo
+ import Ecto.Query
+
+ def run(_ \\ []) do
+ {:ok, %{rows: rows}} = run_sql("list-funnels-with-dupe-goal-ids")
+ data = Enum.map(rows, &to_map/1)
+
+ by_site_id = Enum.group_by(data, & &1.site_id)
+
+ Enum.into(by_site_id, %{}, fn {site_id, data} ->
+ site_meta =
+ %{}
+ |> Map.put(:domain, List.first(data).domain)
+ |> Map.put(:site_id, site_id)
+ |> Map.put(
+ :changes,
+ Enum.map(
+ data,
+ fn d ->
+ %{
+ site_id: site_id,
+ og_goal_id: d.goal_id,
+ goal_name: d.goal_name,
+ funnel_name: d.funnel_name,
+ funnel_id: d.funnel_id,
+ funnel_step_id: d.funnel_step_id,
+ to: Enum.min(d.dupe_goal_ids),
+ og_steps_count: length(d.all_goal_ids)
+ }
+ end
+ )
+ |> Enum.group_by(& &1.funnel_id)
+ )
+
+ {site_id, site_meta}
+ end)
+ |> translate_to_db_ops()
+ |> execute()
+ end
+
+ def execute(data) do
+ Plausible.Repo.transaction(fn ->
+ for {site_id, meta} <- data do
+ IO.puts("\n\nProcessing site ID: #{site_id} (#{meta.domain}):")
+
+ for {funnel_id, changes} <- meta.changes do
+ IO.puts(
+ "\nProcessing changes for funnel ID: #{funnel_id} - '#{Enum.at(changes, 0).funnel_name}'"
+ )
+
+ Enum.each(changes, fn change ->
+ apply_change(funnel_id, change)
+ end)
+ end
+
+ IO.puts("\nFinished processing site ID: #{site_id} (#{meta.domain}).")
+ end
+ end)
+ end
+
+ def apply_change(funnel_id, %{update_type: :delete_whole_funnel} = change) do
+ IO.puts(
+ "Deleting whole funnel '#{change.funnel_name}', there is no way to make it functional without duplicates"
+ )
+
+ Plausible.Repo.delete_all(
+ from(f in Plausible.Funnel,
+ where: f.site_id == ^change.site_id,
+ where: f.id == ^funnel_id
+ )
+ )
+ end
+
+ def apply_change(funnel_id, %{update_type: :delete_step} = change) do
+ IO.puts(
+ "Deleting step ID:#{change.funnel_step_id} '#{change.goal_name}' - otherwise there will be duplicates in the funnel."
+ )
+
+ step =
+ Plausible.Repo.get_by!(Plausible.Funnel.Step,
+ funnel_id: funnel_id,
+ id: change.funnel_step_id
+ )
+
+ {:ok, _} = Plausible.Repo.delete(step)
+ end
+
+ def apply_change(
+ _funnel_id,
+ %{update_type: :update_step, og_goal_id: goal_id, to: goal_id} = change
+ ) do
+ IO.puts(
+ "Doing nothing for step '#{change.goal_name}' - the duplicate goal exists, but the update is no-op"
+ )
+
+ :ok
+ end
+
+ def apply_change(funnel_id, %{update_type: :update_step} = change) do
+ IO.puts(
+ "Updating step '#{change.goal_name}' from goal_id:#{change.og_goal_id} to goal_id:#{change.to}"
+ )
+
+ step =
+ Plausible.Repo.get_by!(Plausible.Funnel.Step,
+ funnel_id: funnel_id,
+ id: change.funnel_step_id
+ )
+
+ change = Ecto.Changeset.change(step, goal_id: change.to)
+ {:ok, _} = Plausible.Repo.update(change)
+ end
+
+ def translate_to_db_ops(data) do
+ for {site, meta} <- data do
+ {site, to_db_ops(meta)}
+ end
+ end
+
+ def to_db_ops(meta) do
+ new_changes =
+ for {funnel_id, changes} <- meta.changes do
+ safe_step_updates = changes |> Enum.uniq_by(& &1.to)
+
+ steps_to_delete = changes -- safe_step_updates
+
+ safe_step_updates =
+ safe_step_updates
+ |> Enum.map(&Map.put(&1, :update_type, :update_step))
+
+ changes =
+ Enum.map(steps_to_delete, &Map.put(&1, :update_type, :delete_step)) ++ safe_step_updates
+
+ og_steps_count = Enum.at(changes, 0).og_steps_count
+
+ steps_after_changes = og_steps_count - Enum.count(steps_to_delete)
+
+ if steps_after_changes >= 2 do
+ {funnel_id, changes}
+ else
+ {funnel_id,
+ [
+ Enum.at(changes, 0)
+ |> Map.put(:update_type, :delete_whole_funnel)
+ ]}
+ end
+ end
+
+ Map.put(meta, :changes, new_changes)
+ end
+
+ def to_map(row) do
+ [
+ domain,
+ site_id,
+ funnel_name,
+ funnel_id,
+ funnel_step_id,
+ goal_id,
+ goal_name,
+ dupe_goal_ids,
+ all_goal_ids
+ ] = row
+
+ %{
+ domain: domain,
+ site_id: site_id,
+ funnel_name: funnel_name,
+ funnel_id: funnel_id,
+ funnel_step_id: funnel_step_id,
+ goal_id: goal_id,
+ goal_name: goal_name,
+ dupe_goal_ids: dupe_goal_ids,
+ all_goal_ids: all_goal_ids
+ }
+ end
+end
diff --git a/lib/plausible/funnels.ex b/lib/plausible/funnels.ex
index 80c18171f3..3a4e82824f 100644
--- a/lib/plausible/funnels.ex
+++ b/lib/plausible/funnels.ex
@@ -45,21 +45,27 @@ defmodule Plausible.Funnels do
]
def list(%Plausible.Site{id: site_id}) do
Repo.all(
- from f in Funnel,
+ from(f in Funnel,
inner_join: steps in assoc(f, :steps),
where: f.site_id == ^site_id,
select: %{name: f.name, id: f.id, steps_count: count(steps)},
group_by: f.id,
order_by: [desc: :id]
+ )
)
end
- @spec delete(Plausible.Site.t(), pos_integer()) :: :ok
+ @spec delete(Plausible.Site.t() | pos_integer(), pos_integer()) :: :ok
def delete(%Plausible.Site{id: site_id}, funnel_id) do
+ delete(site_id, funnel_id)
+ end
+
+ def delete(site_id, funnel_id) do
Repo.delete_all(
- from f in Funnel,
+ from(f in Funnel,
where: f.site_id == ^site_id,
where: f.id == ^funnel_id
+ )
)
:ok
@@ -73,7 +79,7 @@ defmodule Plausible.Funnels do
def get(site_id, funnel_id) when is_integer(site_id) and is_integer(funnel_id) do
q =
- from f in Funnel,
+ from(f in Funnel,
where: f.site_id == ^site_id,
where: f.id == ^funnel_id,
inner_join: steps in assoc(f, :steps),
@@ -82,6 +88,7 @@ defmodule Plausible.Funnels do
preload: [
steps: {steps, goal: goal}
]
+ )
Repo.one(q)
end
diff --git a/priv/data_migrations/FunnelDupeGoals/sql/list-funnels-with-dupe-goal-ids.sql.eex b/priv/data_migrations/FunnelDupeGoals/sql/list-funnels-with-dupe-goal-ids.sql.eex
new file mode 100644
index 0000000000..4ea2339c1c
--- /dev/null
+++ b/priv/data_migrations/FunnelDupeGoals/sql/list-funnels-with-dupe-goal-ids.sql.eex
@@ -0,0 +1,62 @@
+WITH DuplicateGoals AS (
+ SELECT
+ g.site_id,
+ CASE
+ WHEN g.event_name IS NOT NULL THEN 'Event: ' || g.event_name
+ WHEN g.page_path IS NOT NULL THEN 'Page: ' || g.page_path
+ END AS goal_name,
+ ARRAY_AGG(g.id) AS duplicate_ids
+ FROM
+ goals g
+ WHERE
+ g.site_id IS NOT NULL
+ GROUP BY
+ g.site_id,
+ goal_name
+ HAVING COUNT(*) > 1
+),
+FunnelStepsWithDuplicateGoals AS (
+ SELECT
+ fs.id,
+ fs.goal_id,
+ fs.funnel_id,
+ f.name,
+ CASE
+ WHEN g.event_name IS NOT NULL THEN 'Event: ' || g.event_name
+ WHEN g.page_path IS NOT NULL THEN 'Page: ' || g.page_path
+ END AS goal_name,
+ dg.duplicate_ids
+ FROM
+ funnel_steps fs
+ JOIN goals g ON fs.goal_id = g.id
+ JOIN funnels f ON fs.funnel_id = f.id
+ JOIN DuplicateGoals dg ON f.site_id = dg.site_id
+ WHERE
+ fs.goal_id = ANY(dg.duplicate_ids) -- Ch
+)
+SELECT
+ s.domain,
+ f.site_id,
+ f.name as funnel_name,
+ f.id,
+ fs.id,
+ fs.goal_id,
+ fs.goal_name,
+ fs.duplicate_ids,
+ array_agg(fs2.goal_id)
+FROM
+ funnels f
+JOIN FunnelStepsWithDuplicateGoals fs ON f.id = fs.funnel_id
+JOIN funnel_steps fs2 ON f.id = fs2.funnel_id
+JOIN sites s ON s.id = f.site_id
+GROUP BY
+ s.domain,
+ f.site_id,
+ f.name,
+ f.id,
+ fs.id,
+ fs.goal_id,
+ fs.goal_name,
+ fs.duplicate_ids
+ORDER BY fs.id ASC
+;
diff --git a/test/plausible/data_migration/rewrite_funnel_dupes_test.exs b/test/plausible/data_migration/rewrite_funnel_dupes_test.exs
new file mode 100644
index 0000000000..d9ad24ae0a
--- /dev/null
+++ b/test/plausible/data_migration/rewrite_funnel_dupes_test.exs
@@ -0,0 +1,124 @@
+defmodule Plausible.DataMigration.RewriteFunnelDupesTest do
+ use Plausible.DataCase, async: true
+
+ alias Plausible.DataMigration.RewriteFunnelDupes
+
+ import ExUnit.CaptureIO
+
+ for goal_type <- ["event_name", "page_path"] do
+ test "deletes a funnel that cannot be cleaned up from dupe goals (#{goal_type})" do
+ site = insert(:site)
+
+ goals =
+ setup_goals!(site, [
+ %{unquote(goal_type) => "/AAA"},
+ %{unquote(goal_type) => "/AAA"},
+ %{unquote(goal_type) => "/AAA"}
+ ])
+
+ funnel = setup_funnel!(site, "test", goals)
+
+ io = assert capture_io(fn -> RewriteFunnelDupes.run() end)
+ assert io =~ "Processing site ID: #{site.id}"
+ assert io =~ "Deleting whole funnel"
+
+ refute Repo.reload(funnel)
+ end
+
+ test "reduces a funnel if possible (#{goal_type})" do
+ site = insert(:site)
+
+ goals =
+ setup_goals!(site, [
+ %{unquote(goal_type) => "/AAA"},
+ %{unquote(goal_type) => "/BBB"},
+ %{unquote(goal_type) => "/AAA"},
+ %{unquote(goal_type) => "/CCC"}
+ ])
+
+ funnel = setup_funnel!(site, "test", goals)
+
+ io = assert capture_io(fn -> RewriteFunnelDupes.run() end)
+ assert io =~ "Processing site ID: #{site.id}"
+ assert io =~ "Deleting step"
+
+ assert_funnel(site.id, funnel.id, [
+ %{unquote(goal_type) => "/AAA"},
+ %{unquote(goal_type) => "/BBB"},
+ %{unquote(goal_type) => "/CCC"}
+ ])
+ end
+ end
+
+ test "dupe names in mixed goal types don't matter" do
+ site = insert(:site)
+
+ gs = [
+ %{"event_name" => "/AAA"},
+ %{"page_path" => "/AAA"},
+ %{"event_name" => "/foo"},
+ %{"page_path" => "/foo"}
+ ]
+
+ goals = setup_goals!(site, gs)
+
+ funnel = setup_funnel!(site, "test", goals)
+
+ capture_io(fn -> RewriteFunnelDupes.run() end)
+
+ assert_funnel(site.id, funnel.id, gs)
+ end
+
+ test "goals across multiple funnels" do
+ site = insert(:site)
+
+ goals = [
+ %{"event_name" => "/AAA"},
+ %{"event_name" => "/AAA"},
+ %{"page_path" => "/foo"},
+ %{"event_name" => "/AAA"}
+ ]
+
+ [g1, g2, g3, g4] = setup_goals!(site, goals)
+
+ funnel1 = setup_funnel!(site, "test1", [g1, g2, g3, g4])
+ funnel2 = setup_funnel!(site, "test2", [g2, g1, g3, g4])
+ funnel3 = setup_funnel!(site, "test3", [g2, g3])
+ funnel4 = setup_funnel!(site, "test4", [g2, g1])
+
+ capture_io(fn -> RewriteFunnelDupes.run() end)
+
+ assert_funnel(site.id, funnel1.id, [%{"event_name" => "/AAA"}, %{"page_path" => "/foo"}])
+ assert_funnel(site.id, funnel2.id, [%{"event_name" => "/AAA"}, %{"page_path" => "/foo"}])
+ assert_funnel(site.id, funnel3.id, [%{"event_name" => "/AAA"}, %{"page_path" => "/foo"}])
+
+ refute Plausible.Funnels.get(site.id, funnel4.id)
+ end
+
+ defp setup_goals!(site, goals) do
+ for goal <- goals do
+ {:ok, g} = Plausible.Goals.create(site, goal)
+ g
+ end
+ end
+
+ def setup_funnel!(site, name, goals) do
+ {:ok, funnel} = Plausible.Funnels.create(site, name, Enum.map(goals, &%{"goal_id" => &1.id}))
+ funnel
+ end
+
+ def assert_funnel(site_id, funnel_id, goals) do
+ funnel_goals =
+ Plausible.Funnels.get(site_id, funnel_id)
+ |> Map.fetch!(:steps)
+ |> Enum.map(& &1.goal)
+
+ assert Enum.count(funnel_goals) == Enum.count(goals)
+
+ Enum.zip(funnel_goals, goals)
+ |> Enum.each(fn {funnel_goal, expect} ->
+ key = Map.keys(expect) |> List.first() |> String.to_existing_atom()
+ assert Map.fetch!(funnel_goal, key) == Map.values(expect) |> List.first()
+ end)
+ end
+end