Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Data Migration: rewrite funnels that contain duplicate goals (#3361)
hq1542385feabbb
diff --git a/lib/plausible/data_migration/rewrite_funnel_dupes.ex b/lib/plausible/data_migration/rewrite_funnel_dupes.exnew file mode 100644index 0000000000..52db56d347--- /dev/null+++ b/lib/plausible/data_migration/rewrite_funnel_dupes.ex@@ -0,0 +1,191 @@+defmodule Plausible.DataMigration.RewriteFunnelDupes do+ @moduledoc """+ A data fix migration that seeks funnels having steps+ whose goals are equally named.+ It then tries to rewrite the duplicate goals using the+ oldest goal available. In extreme cases, e.g. when multiple+ duplicates are found for a single funnel, it will either+ reduce or completely remove the funnel.+ This enables us to run a migration later on that will+ delete duplicate goals and enforce goal uniqueness at the+ database level via a CHECK constraint.++ To run, just call the `run` function.+ """+ use Plausible.DataMigration, dir: "FunnelDupeGoals", repo: Plausible.Repo+ import Ecto.Query++ def run(_ \\ []) do+ {:ok, %{rows: rows}} = run_sql("list-funnels-with-dupe-goal-ids")+ data = Enum.map(rows, &to_map/1)++ by_site_id = Enum.group_by(data, & &1.site_id)++ Enum.into(by_site_id, %{}, fn {site_id, data} ->+ site_meta =+ %{}+ |> Map.put(:domain, List.first(data).domain)+ |> Map.put(:site_id, site_id)+ |> Map.put(+ :changes,+ Enum.map(+ data,+ fn d ->+ %{+ site_id: site_id,+ og_goal_id: d.goal_id,+ goal_name: d.goal_name,+ funnel_name: d.funnel_name,+ funnel_id: d.funnel_id,+ funnel_step_id: d.funnel_step_id,+ to: Enum.min(d.dupe_goal_ids),+ og_steps_count: length(d.all_goal_ids)+ }+ end+ )+ |> Enum.group_by(& &1.funnel_id)+ )++ {site_id, site_meta}+ end)+ |> translate_to_db_ops()+ |> execute()+ end++ def execute(data) do+ Plausible.Repo.transaction(fn ->+ for {site_id, meta} <- data do+ IO.puts("\n\nProcessing site ID: #{site_id} (#{meta.domain}):")++ for {funnel_id, changes} <- meta.changes do+ IO.puts(+ "\nProcessing changes for funnel ID: #{funnel_id} - '#{Enum.at(changes, 0).funnel_name}'"+ )++ Enum.each(changes, fn change ->+ apply_change(funnel_id, change)+ end)+ end++ IO.puts("\nFinished processing site ID: #{site_id} (#{meta.domain}).")+ end+ end)+ end++ def apply_change(funnel_id, %{update_type: :delete_whole_funnel} = change) do+ IO.puts(+ "Deleting whole funnel '#{change.funnel_name}', there is no way to make it functional without duplicates"+ )++ Plausible.Repo.delete_all(+ from(f in Plausible.Funnel,+ where: f.site_id == ^change.site_id,+ where: f.id == ^funnel_id+ )+ )+ end++ def apply_change(funnel_id, %{update_type: :delete_step} = change) do+ IO.puts(+ "Deleting step ID:#{change.funnel_step_id} '#{change.goal_name}' - otherwise there will be duplicates in the funnel."+ )++ step =+ Plausible.Repo.get_by!(Plausible.Funnel.Step,+ funnel_id: funnel_id,+ id: change.funnel_step_id+ )++ {:ok, _} = Plausible.Repo.delete(step)+ end++ def apply_change(+ _funnel_id,+ %{update_type: :update_step, og_goal_id: goal_id, to: goal_id} = change+ ) do+ IO.puts(+ "Doing nothing for step '#{change.goal_name}' - the duplicate goal exists, but the update is no-op"+ )++ :ok+ end++ def apply_change(funnel_id, %{update_type: :update_step} = change) do+ IO.puts(+ "Updating step '#{change.goal_name}' from goal_id:#{change.og_goal_id} to goal_id:#{change.to}"+ )++ step =+ Plausible.Repo.get_by!(Plausible.Funnel.Step,+ funnel_id: funnel_id,+ id: change.funnel_step_id+ )++ change = Ecto.Changeset.change(step, goal_id: change.to)+ {:ok, _} = Plausible.Repo.update(change)+ end++ def translate_to_db_ops(data) do+ for {site, meta} <- data do+ {site, to_db_ops(meta)}+ end+ end++ def to_db_ops(meta) do+ new_changes =+ for {funnel_id, changes} <- meta.changes do+ safe_step_updates = changes |> Enum.uniq_by(& &1.to)++ steps_to_delete = changes -- safe_step_updates++ safe_step_updates =+ safe_step_updates+ |> Enum.map(&Map.put(&1, :update_type, :update_step))++ changes =+ Enum.map(steps_to_delete, &Map.put(&1, :update_type, :delete_step)) ++ safe_step_updates++ og_steps_count = Enum.at(changes, 0).og_steps_count++ steps_after_changes = og_steps_count - Enum.count(steps_to_delete)++ if steps_after_changes >= 2 do+ {funnel_id, changes}+ else+ {funnel_id,+ [+ Enum.at(changes, 0)+ |> Map.put(:update_type, :delete_whole_funnel)+ ]}+ end+ end++ Map.put(meta, :changes, new_changes)+ end++ def to_map(row) do+ [+ domain,+ site_id,+ funnel_name,+ funnel_id,+ funnel_step_id,+ goal_id,+ goal_name,+ dupe_goal_ids,+ all_goal_ids+ ] = row++ %{+ domain: domain,+ site_id: site_id,+ funnel_name: funnel_name,+ funnel_id: funnel_id,+ funnel_step_id: funnel_step_id,+ goal_id: goal_id,+ goal_name: goal_name,+ dupe_goal_ids: dupe_goal_ids,+ all_goal_ids: all_goal_ids+ }+ end+enddiff --git a/lib/plausible/funnels.ex b/lib/plausible/funnels.exindex 80c18171f3..3a4e82824f 100644--- a/lib/plausible/funnels.ex+++ b/lib/plausible/funnels.ex@@ -45,21 +45,27 @@ defmodule Plausible.Funnels do]def list(%Plausible.Site{id: site_id}) doRepo.all(- from f in Funnel,+ from(f in Funnel,inner_join: steps in assoc(f, :steps),where: f.site_id == ^site_id,select: %{name: f.name, id: f.id, steps_count: count(steps)},group_by: f.id,order_by: [desc: :id]+ ))end- @spec delete(Plausible.Site.t(), pos_integer()) :: :ok+ @spec delete(Plausible.Site.t() | pos_integer(), pos_integer()) :: :okdef delete(%Plausible.Site{id: site_id}, funnel_id) do+ delete(site_id, funnel_id)+ end++ def delete(site_id, funnel_id) doRepo.delete_all(- from f in Funnel,+ from(f in Funnel,where: f.site_id == ^site_id,where: f.id == ^funnel_id+ )):ok@@ -73,7 +79,7 @@ defmodule Plausible.Funnels dodef get(site_id, funnel_id) when is_integer(site_id) and is_integer(funnel_id) doq =- from f in Funnel,+ from(f in Funnel,where: f.site_id == ^site_id,where: f.id == ^funnel_id,inner_join: steps in assoc(f, :steps),@@ -82,6 +88,7 @@ defmodule Plausible.Funnels dopreload: [steps: {steps, goal: goal}]+ )Repo.one(q)enddiff --git a/priv/data_migrations/FunnelDupeGoals/sql/list-funnels-with-dupe-goal-ids.sql.eex b/priv/data_migrations/FunnelDupeGoals/sql/list-funnels-with-dupe-goal-ids.sql.eexnew file mode 100644index 0000000000..4ea2339c1c--- /dev/null+++ b/priv/data_migrations/FunnelDupeGoals/sql/list-funnels-with-dupe-goal-ids.sql.eex@@ -0,0 +1,62 @@+WITH DuplicateGoals AS (+ SELECT+ g.site_id,+ CASE+ WHEN g.event_name IS NOT NULL THEN 'Event: ' || g.event_name+ WHEN g.page_path IS NOT NULL THEN 'Page: ' || g.page_path+ END AS goal_name,+ ARRAY_AGG(g.id) AS duplicate_ids+ FROM+ goals g+ WHERE+ g.site_id IS NOT NULL+ GROUP BY+ g.site_id,+ goal_name+ HAVING COUNT(*) > 1+),+FunnelStepsWithDuplicateGoals AS (+ SELECT+ fs.id,+ fs.goal_id,+ fs.funnel_id,+ f.name,+ CASE+ WHEN g.event_name IS NOT NULL THEN 'Event: ' || g.event_name+ WHEN g.page_path IS NOT NULL THEN 'Page: ' || g.page_path+ END AS goal_name,+ dg.duplicate_ids+ FROM+ funnel_steps fs+ JOIN goals g ON fs.goal_id = g.id+ JOIN funnels f ON fs.funnel_id = f.id+ JOIN DuplicateGoals dg ON f.site_id = dg.site_id+ WHERE+ fs.goal_id = ANY(dg.duplicate_ids) -- Ch+)+SELECT+ s.domain,+ f.site_id,+ f.name as funnel_name,+ f.id,+ fs.id,+ fs.goal_id,+ fs.goal_name,+ fs.duplicate_ids,+ array_agg(fs2.goal_id)+FROM+ funnels f+JOIN FunnelStepsWithDuplicateGoals fs ON f.id = fs.funnel_id+JOIN funnel_steps fs2 ON f.id = fs2.funnel_id+JOIN sites s ON s.id = f.site_id+GROUP BY+ s.domain,+ f.site_id,+ f.name,+ f.id,+ fs.id,+ fs.goal_id,+ fs.goal_name,+ fs.duplicate_ids+ORDER BY fs.id ASC+;diff --git a/test/plausible/data_migration/rewrite_funnel_dupes_test.exs b/test/plausible/data_migration/rewrite_funnel_dupes_test.exsnew file mode 100644index 0000000000..d9ad24ae0a--- /dev/null+++ b/test/plausible/data_migration/rewrite_funnel_dupes_test.exs@@ -0,0 +1,124 @@+defmodule Plausible.DataMigration.RewriteFunnelDupesTest do+ use Plausible.DataCase, async: true++ alias Plausible.DataMigration.RewriteFunnelDupes++ import ExUnit.CaptureIO++ for goal_type <- ["event_name", "page_path"] do+ test "deletes a funnel that cannot be cleaned up from dupe goals (#{goal_type})" do+ site = insert(:site)++ goals =+ setup_goals!(site, [+ %{unquote(goal_type) => "/AAA"},+ %{unquote(goal_type) => "/AAA"},+ %{unquote(goal_type) => "/AAA"}+ ])++ funnel = setup_funnel!(site, "test", goals)++ io = assert capture_io(fn -> RewriteFunnelDupes.run() end)+ assert io =~ "Processing site ID: #{site.id}"+ assert io =~ "Deleting whole funnel"++ refute Repo.reload(funnel)+ end++ test "reduces a funnel if possible (#{goal_type})" do+ site = insert(:site)++ goals =+ setup_goals!(site, [+ %{unquote(goal_type) => "/AAA"},+ %{unquote(goal_type) => "/BBB"},+ %{unquote(goal_type) => "/AAA"},+ %{unquote(goal_type) => "/CCC"}+ ])++ funnel = setup_funnel!(site, "test", goals)++ io = assert capture_io(fn -> RewriteFunnelDupes.run() end)+ assert io =~ "Processing site ID: #{site.id}"+ assert io =~ "Deleting step"++ assert_funnel(site.id, funnel.id, [+ %{unquote(goal_type) => "/AAA"},+ %{unquote(goal_type) => "/BBB"},+ %{unquote(goal_type) => "/CCC"}+ ])+ end+ end++ test "dupe names in mixed goal types don't matter" do+ site = insert(:site)++ gs = [+ %{"event_name" => "/AAA"},+ %{"page_path" => "/AAA"},+ %{"event_name" => "/foo"},+ %{"page_path" => "/foo"}+ ]++ goals = setup_goals!(site, gs)++ funnel = setup_funnel!(site, "test", goals)++ capture_io(fn -> RewriteFunnelDupes.run() end)++ assert_funnel(site.id, funnel.id, gs)+ end++ test "goals across multiple funnels" do+ site = insert(:site)++ goals = [+ %{"event_name" => "/AAA"},+ %{"event_name" => "/AAA"},+ %{"page_path" => "/foo"},+ %{"event_name" => "/AAA"}+ ]++ [g1, g2, g3, g4] = setup_goals!(site, goals)++ funnel1 = setup_funnel!(site, "test1", [g1, g2, g3, g4])+ funnel2 = setup_funnel!(site, "test2", [g2, g1, g3, g4])+ funnel3 = setup_funnel!(site, "test3", [g2, g3])+ funnel4 = setup_funnel!(site, "test4", [g2, g1])++ capture_io(fn -> RewriteFunnelDupes.run() end)++ assert_funnel(site.id, funnel1.id, [%{"event_name" => "/AAA"}, %{"page_path" => "/foo"}])+ assert_funnel(site.id, funnel2.id, [%{"event_name" => "/AAA"}, %{"page_path" => "/foo"}])+ assert_funnel(site.id, funnel3.id, [%{"event_name" => "/AAA"}, %{"page_path" => "/foo"}])++ refute Plausible.Funnels.get(site.id, funnel4.id)+ end++ defp setup_goals!(site, goals) do+ for goal <- goals do+ {:ok, g} = Plausible.Goals.create(site, goal)+ g+ end+ end++ def setup_funnel!(site, name, goals) do+ {:ok, funnel} = Plausible.Funnels.create(site, name, Enum.map(goals, &%{"goal_id" => &1.id}))+ funnel+ end++ def assert_funnel(site_id, funnel_id, goals) do+ funnel_goals =+ Plausible.Funnels.get(site_id, funnel_id)+ |> Map.fetch!(:steps)+ |> Enum.map(& &1.goal)++ assert Enum.count(funnel_goals) == Enum.count(goals)++ Enum.zip(funnel_goals, goals)+ |> Enum.each(fn {funnel_goal, expect} ->+ key = Map.keys(expect) |> List.first() |> String.to_existing_atom()+ assert Map.fetch!(funnel_goal, key) == Map.values(expect) |> List.first()+ end)+ end+end