Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Delete imports which have no stats (#3972)
Adrian Gruntkowski33eed9d7db2b
diff --git a/lib/plausible/data_migration/site_imports.ex b/lib/plausible/data_migration/site_imports.exindex 5c84b642a2..fc4c91b2d8 100644--- a/lib/plausible/data_migration/site_imports.ex+++ b/lib/plausible/data_migration/site_imports.ex@@ -72,35 +72,36 @@ defmodule Plausible.DataMigration.SiteImports doend_date = imported_stats_end_date(site.id, import_ids)- end_date =- if !end_date do- IO.puts(- "Site import #{site_import.id} (site ID #{site.id}) does not have any recorded stats. Setting end date to minimum."- )+ if !end_date do+ IO.puts(+ "Site import #{site_import.id} (site ID #{site.id}) does not have any recorded stats. Removing it."+ )- Date.add(site_import.start_date, 2)- else- end_date- end-- end_date =- if Date.compare(end_date, site_import.end_date) in [:lt, :eq] do- end_date- else- IO.puts(- "Site import #{site_import.id} (site ID #{site.id}) computed end date is later than the current one. Skipping."- )-- site_import.end_date+ if site_import.legacy do+ clear_imported_data(site, dry_run?)end- site_import- |> Ecto.Changeset.change(end_date: end_date)- |> update!(dry_run?)-- IO.puts(- "End date of site import #{site_import.id} (site ID #{site.id}) adjusted to #{end_date}"- )+ delete!(site_import, dry_run?)+ else+ end_date =+ if Date.compare(end_date, site_import.end_date) in [:lt, :eq] do+ end_date+ else+ IO.puts(+ "Site import #{site_import.id} (site ID #{site.id}) computed end date is later than the current one. Skipping."+ )++ site_import.end_date+ end++ site_import+ |> Ecto.Changeset.change(end_date: end_date)+ |> update!(dry_run?)++ IO.puts(+ "End date of site import #{site_import.id} (site ID #{site.id}) adjusted to #{end_date}"+ )+ endendIO.puts("Done processing site ID #{site.id}")@@ -149,6 +150,14 @@ defmodule Plausible.DataMigration.SiteImports doendend+ defp clear_imported_data(site, false = _dry_run?) do+ Repo.update_all(from(s in Site, where: s.id == ^site.id), set: [imported_data: nil])+ end++ defp clear_imported_data(site, true = _dry_run?) do+ %{site | imported_data: nil}+ end+defp update!(changeset, false = _dry_run?) doRepo.update!(changeset)end@@ -161,6 +170,14 @@ defmodule Plausible.DataMigration.SiteImports doendend+ defp delete!(entity, false = _dry_run?) do+ Repo.delete!(entity)+ end++ defp delete!(entity, true = _dry_run?) do+ entity+ end+defp max_date_query(schema, site_id, import_ids) dofrom(q in schema,where: q.site_id == ^site_id,diff --git a/test/plausible/data_migration/site_imports_test.exs b/test/plausible/data_migration/site_imports_test.exsindex c8e48d0d3a..a99ded0976 100644--- a/test/plausible/data_migration/site_imports_test.exs+++ b/test/plausible/data_migration/site_imports_test.exs@@ -91,6 +91,22 @@ defmodule Plausible.DataMigration.SiteImportsTest doassert site_import.source == :universal_analyticsend+ test "removes site import when there are no stats" do+ site =+ insert(:site)+ |> Site.start_import(~D[2021-01-02], ~D[2020-02-02], "Google Analytics", "ok")+ |> Repo.update!()++ _another_site_import = insert(:site_import, site: site)++ assert capture_io(fn ->+ assert :ok = SiteImports.run(dry_run?: false)+ end) =~ "Processing 1 site"++ site = Repo.reload!(site)+ assert [] = Imported.list_all_imports(site)+ end+test "leaves site and imports unchanged if everything fits" dosite =insert(:site)