Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Ensure only complete imports are considered in site imports data migration (#3987)
Adrian Gruntkowski1c1ea95e161a
diff --git a/lib/plausible/data_migration/site_imports.ex b/lib/plausible/data_migration/site_imports.exindex 53f06947b4..eddc0d8739 100644--- a/lib/plausible/data_migration/site_imports.ex+++ b/lib/plausible/data_migration/site_imports.ex@@ -20,50 +20,73 @@ defmodule Plausible.DataMigration.SiteImports dodry_run? = Keyword.get(opts, :dry_run?, true)site_import_query =- from(i in Imported.SiteImport, where: i.site_id == parent_as(:site).id, select: 1)+ from(i in Imported.SiteImport,+ where: i.site_id == parent_as(:site).id and i.status == ^SiteImport.completed(),+ select: 1+ )++ sites_with_only_legacy_import =+ from(s in Site,+ as: :site,+ where:+ not is_nil(s.imported_data) and fragment("?->>'status'", s.imported_data) == "ok" and+ not exists(site_import_query)+ )+ |> Repo.all(log: false)++ backfill_legacy_site_imports(sites_with_only_legacy_import, dry_run?)- sites_with_imports =- from(s in Site, as: :site, where: not is_nil(s.imported_data) or exists(site_import_query))+ sites_with_site_imports =+ from(s in Site,+ as: :site,+ where: exists(site_import_query)+ )|> Repo.all(log: false)- sites_count = length(sites_with_imports)+ adjust_site_import_end_dates(sites_with_site_imports, dry_run?)++ IO.puts("Finished")+ end++ defp backfill_legacy_site_imports(sites, dry_run?) do+ total = length(sites)++ IO.puts("Backfilling legacy site import across #{total} sites (DRY RUN: #{dry_run?})...")++ for {site, idx} <- Enum.with_index(sites) do+ IO.puts("Creating legacy site import entry for site ID #{site.id} (#{idx + 1}/#{total})")- IO.puts("Processing #{sites_count} sites with imports (DRY RUN: #{dry_run?})...")+ params =+ site.imported_data+ |> Imported.SiteImport.from_legacy()+ |> Map.put(:site_id, site.id)+ |> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id])- for {site, idx} <- Enum.with_index(sites_with_imports) do+ %Imported.SiteImport{}+ |> Ecto.Changeset.change(params)+ |> insert!(dry_run?)+ end++ IO.puts("Finished backfilling sites.")+ end++ defp adjust_site_import_end_dates(sites, dry_run?) do+ total = length(sites)++ IO.puts("Adjusting end dates of site imports across #{total} sites (DRY RUN: #{dry_run?})...")++ for {site, idx} <- Enum.with_index(sites) dosite_imports =+ [_ | _] =from(i in Imported.SiteImport,where: i.site_id == ^site.id and i.status == ^SiteImport.completed())|> Repo.all(log: false)IO.puts(- "Processing site ID #{site.id} (#{idx + 1} / #{sites_count}) (imported_data: #{is_struct(site.imported_data)}, site_imports: #{length(site_imports)})"+ "Processing site ID #{site.id} (#{idx + 1} / #{total}) (imported_data: #{is_struct(site.imported_data)}, site_imports: #{length(site_imports)})")- site_imports =- if site.imported_data && not Enum.any?(site_imports, & &1.legacy) do- IO.puts("Creating legacy site import entry for site ID #{site.id}")-- # create legacy entry if there's not one yet- params =- site.imported_data- |> Imported.SiteImport.from_legacy()- |> Map.put(:site_id, site.id)- |> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id])-- legacy_site_import =- %Imported.SiteImport{}- |> Ecto.Changeset.change(params)- |> insert!(dry_run?)-- [legacy_site_import | site_imports]- else- IO.puts("Legacy site import entry for site ID #{site.id} already exists")-- site_imports- end-# adjust end date for each site importfor site_import <- site_imports doIO.puts(@@ -129,7 +152,7 @@ defmodule Plausible.DataMigration.SiteImports doIO.puts("Done processing site ID #{site.id}")end- IO.puts("Finished")+ IO.puts("Finished adjusting end dates of site imports.")end# Exposed for testing purposesdiff --git a/test/plausible/data_migration/site_imports_test.exs b/test/plausible/data_migration/site_imports_test.exsindex e1bf730f96..19a80efcf1 100644--- a/test/plausible/data_migration/site_imports_test.exs+++ b/test/plausible/data_migration/site_imports_test.exs@@ -15,7 +15,7 @@ defmodule Plausible.DataMigration.SiteImportsTest doassert :ok = SiteImports.run()end)- assert dry_run_output =~ "Processing 0 sites"+ assert dry_run_output =~ "Backfilling legacy site import across 0 sites"assert dry_run_output =~ "DRY RUN: true"real_run_output =@@ -23,7 +23,7 @@ defmodule Plausible.DataMigration.SiteImportsTest doassert :ok = SiteImports.run(dry_run?: false)end)- assert real_run_output =~ "Processing 0 sites"+ assert real_run_output =~ "Backfilling legacy site import across 0 sites"assert real_run_output =~ "DRY RUN: false"end@@ -39,7 +39,7 @@ defmodule Plausible.DataMigration.SiteImportsTest doassert capture_io(fn ->assert :ok = SiteImports.run(dry_run?: false)- end) =~ "Processing 1 sites"+ end) =~ "Backfilling legacy site import across 1 sites"site = Repo.reload!(site)@@ -63,7 +63,7 @@ defmodule Plausible.DataMigration.SiteImportsTest doassert capture_io(fn ->assert :ok = SiteImports.run()- end) =~ "Processing 1 sites"+ end) =~ "Backfilling legacy site import across 1 sites"site = Repo.reload!(site)@@ -84,7 +84,7 @@ defmodule Plausible.DataMigration.SiteImportsTest doassert capture_io(fn ->assert :ok = SiteImports.run(dry_run?: false)- end) =~ "Processing 1 sites"+ end) =~ "Backfilling legacy site import across 1 sites"site = Repo.reload!(site)@@ -102,11 +102,9 @@ defmodule Plausible.DataMigration.SiteImportsTest do|> Site.start_import(~D[2021-01-02], ~D[2020-02-02], "Google Analytics", "ok")|> Repo.update!()- _another_site_import = insert(:site_import, site: site)-assert capture_io(fn ->assert :ok = SiteImports.run(dry_run?: false)- end) =~ "Processing 1 site"+ end) =~ "Backfilling legacy site import across 1 sites"site = Repo.reload!(site)assert [] = Imported.list_all_imports(site)@@ -134,7 +132,7 @@ defmodule Plausible.DataMigration.SiteImportsTest doassert capture_io(fn ->assert :ok = SiteImports.run(dry_run?: false)- end) =~ "Processing 1 sites"+ end) =~ "Backfilling legacy site import across 0 sites"site = Repo.reload!(site)@@ -144,6 +142,69 @@ defmodule Plausible.DataMigration.SiteImportsTest doassert site_import.end_date == ~D[2021-01-08]assert site_import.source == :universal_analyticsend++ test "only considers sites with completed site imports or 'ok' imported data" do+ site1 =+ insert(:site)+ |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")+ |> Repo.update!()++ existing_import1 =+ insert(:site_import,+ site: site1,+ start_date: ~D[2021-01-02],+ end_date: ~D[2021-01-08],+ status: :completed,+ legacy: false+ )++ site2 =+ insert(:site)+ |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")+ |> Repo.update!()++ existing_import2 =+ insert(:site_import,+ site: site2,+ start_date: ~D[2021-01-02],+ end_date: ~D[2021-01-08],+ status: :failed,+ legacy: false+ )++ site3 =+ insert(:site)+ |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")+ |> Repo.update!()++ site4 =+ insert(:site)+ |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")+ |> Repo.update!()++ _existing_import3 =+ insert(:site_import,+ site: site4,+ start_date: ~D[2021-01-02],+ end_date: ~D[2021-01-08],+ status: :failed,+ legacy: true+ )++ output =+ capture_io(fn ->+ assert :ok = SiteImports.run(dry_run?: false)+ end)++ assert output =~ "Backfilling legacy site import across 1 sites"+ assert output =~ "Adjusting end dates of site imports across 2 sites"+ assert output =~ "site ID #{site1.id} "+ assert output =~ "site import #{existing_import1.id} "+ assert output =~ "site ID #{site2.id} "+ refute output =~ "site import #{existing_import2.id} "+ refute output =~ "site ID #{site3.id} "+ refute output =~ "site ID #{site4.id} "+ endenddescribe "imported_stats_end_date/1" do