Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Keep `sites.imported_data` in sync with backfilled `SiteImport` when migrating (#3979)
Adrian Gruntkowskid79678871557
diff --git a/lib/plausible/data_migration/site_imports.ex b/lib/plausible/data_migration/site_imports.exindex fc4c91b2d8..53f06947b4 100644--- a/lib/plausible/data_migration/site_imports.ex+++ b/lib/plausible/data_migration/site_imports.ex@@ -2,15 +2,20 @@ defmodule Plausible.DataMigration.SiteImports do@moduledoc """Site imports migration backfilling SiteImport entries for old importsand alters import end dates to match actual end date of respective import stats.++"""import Ecto.Queryalias Plausible.ClickhouseRepoalias Plausible.Imported+ alias Plausible.Imported.SiteImportalias Plausible.Repoalias Plausible.Site+ require Plausible.Imported.SiteImport+def run(opts \\ []) dodry_run? = Keyword.get(opts, :dry_run?, true)@@ -27,7 +32,9 @@ defmodule Plausible.DataMigration.SiteImports dofor {site, idx} <- Enum.with_index(sites_with_imports) dosite_imports =- from(i in Imported.SiteImport, where: i.site_id == ^site.id)+ from(i in Imported.SiteImport,+ where: i.site_id == ^site.id and i.status == ^SiteImport.completed()+ )|> Repo.all(log: false)IO.puts(@@ -78,29 +85,44 @@ defmodule Plausible.DataMigration.SiteImports do)if site_import.legacy do+ # sanity check that data is correct+ "ok" = site.imported_data.status+clear_imported_data(site, dry_run?)enddelete!(site_import, dry_run?)else- end_date =- if Date.compare(end_date, site_import.end_date) in [:lt, :eq] do- end_date- else+ case Date.compare(end_date, site_import.end_date) do+ :lt ->IO.puts(- "Site import #{site_import.id} (site ID #{site.id}) computed end date is later than the current one. Skipping."+ "End date of site import #{site_import.id} (site ID #{site.id}) is adjusted from #{site_import.end_date} to #{end_date}.")- site_import.end_date- end+ site_import+ |> Ecto.Changeset.change(end_date: end_date)+ |> update!(dry_run?)- site_import- |> Ecto.Changeset.change(end_date: end_date)- |> update!(dry_run?)+ # credo:disable-for-next-line Credo.Check.Refactor.Nesting+ if site_import.legacy do+ # sanity check that data is correct+ "ok" = site.imported_data.status- IO.puts(- "End date of site import #{site_import.id} (site ID #{site.id}) adjusted to #{end_date}"- )+ site+ |> Ecto.Changeset.change(imported_data: %{end_date: end_date})+ |> update!(dry_run?)+ end++ :eq ->+ IO.puts(+ "End date of site import #{site_import.id} (site ID #{site.id}) is left unadjusted."+ )++ :gt ->+ IO.puts(+ "Site import #{site_import.id} (site ID #{site.id}) computed end date is later than the current one. Skipping."+ )+ endendenddiff --git a/test/plausible/data_migration/site_imports_test.exs b/test/plausible/data_migration/site_imports_test.exsindex a99ded0976..e1bf730f96 100644--- a/test/plausible/data_migration/site_imports_test.exs+++ b/test/plausible/data_migration/site_imports_test.exs@@ -47,6 +47,7 @@ defmodule Plausible.DataMigration.SiteImportsTest doassert id > 0assert site_import.start_date == site.imported_data.start_dateassert site_import.end_date == ~D[2021-01-07]+ assert site.imported_data.end_date == ~D[2021-01-07]assert site_import.source == :universal_analyticsend@@ -64,8 +65,11 @@ defmodule Plausible.DataMigration.SiteImportsTest doassert :ok = SiteImports.run()end) =~ "Processing 1 sites"+ site = Repo.reload!(site)+assert [%{id: id, legacy: true}] = Imported.list_all_imports(site)assert id == 0+ assert site.imported_data.end_date == ~D[2021-01-08]endtest "does not set end date to latter than the current one" do@@ -88,6 +92,7 @@ defmodule Plausible.DataMigration.SiteImportsTest doassert id > 0assert site_import.start_date == site.imported_data.start_dateassert site_import.end_date == ~D[2021-01-08]+ assert site.imported_data.end_date == ~D[2021-01-08]assert site_import.source == :universal_analyticsend@@ -105,6 +110,7 @@ defmodule Plausible.DataMigration.SiteImportsTest dosite = Repo.reload!(site)assert [] = Imported.list_all_imports(site)+ assert site.imported_data == nilendtest "leaves site and imports unchanged if everything fits" do