Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Ensure only complete imports are considered in site imports data migration (#3987)

Adrian Gruntkowski1c1ea95e161a
diff --git a/lib/plausible/data_migration/site_imports.ex b/lib/plausible/data_migration/site_imports.ex
index 53f06947b4..eddc0d8739 100644
--- a/lib/plausible/data_migration/site_imports.ex
+++ b/lib/plausible/data_migration/site_imports.ex
@@ -20,50 +20,73 @@ defmodule Plausible.DataMigration.SiteImports do
dry_run? = Keyword.get(opts, :dry_run?, true)
site_import_query =
- from(i in Imported.SiteImport, where: i.site_id == parent_as(:site).id, select: 1)
+ from(i in Imported.SiteImport,
+ where: i.site_id == parent_as(:site).id and i.status == ^SiteImport.completed(),
+ select: 1
+ )
+
+ sites_with_only_legacy_import =
+ from(s in Site,
+ as: :site,
+ where:
+ not is_nil(s.imported_data) and fragment("?->>'status'", s.imported_data) == "ok" and
+ not exists(site_import_query)
+ )
+ |> Repo.all(log: false)
+
+ backfill_legacy_site_imports(sites_with_only_legacy_import, dry_run?)
- sites_with_imports =
- from(s in Site, as: :site, where: not is_nil(s.imported_data) or exists(site_import_query))
+ sites_with_site_imports =
+ from(s in Site,
+ as: :site,
+ where: exists(site_import_query)
+ )
|> Repo.all(log: false)
- sites_count = length(sites_with_imports)
+ adjust_site_import_end_dates(sites_with_site_imports, dry_run?)
+
+ IO.puts("Finished")
+ end
+
+ defp backfill_legacy_site_imports(sites, dry_run?) do
+ total = length(sites)
+
+ IO.puts("Backfilling legacy site import across #{total} sites (DRY RUN: #{dry_run?})...")
+
+ for {site, idx} <- Enum.with_index(sites) do
+ IO.puts("Creating legacy site import entry for site ID #{site.id} (#{idx + 1}/#{total})")
- IO.puts("Processing #{sites_count} sites with imports (DRY RUN: #{dry_run?})...")
+ params =
+ site.imported_data
+ |> Imported.SiteImport.from_legacy()
+ |> Map.put(:site_id, site.id)
+ |> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id])
- for {site, idx} <- Enum.with_index(sites_with_imports) do
+ %Imported.SiteImport{}
+ |> Ecto.Changeset.change(params)
+ |> insert!(dry_run?)
+ end
+
+ IO.puts("Finished backfilling sites.")
+ end
+
+ defp adjust_site_import_end_dates(sites, dry_run?) do
+ total = length(sites)
+
+ IO.puts("Adjusting end dates of site imports across #{total} sites (DRY RUN: #{dry_run?})...")
+
+ for {site, idx} <- Enum.with_index(sites) do
site_imports =
+ [_ | _] =
from(i in Imported.SiteImport,
where: i.site_id == ^site.id and i.status == ^SiteImport.completed()
)
|> Repo.all(log: false)
IO.puts(
- "Processing site ID #{site.id} (#{idx + 1} / #{sites_count}) (imported_data: #{is_struct(site.imported_data)}, site_imports: #{length(site_imports)})"
+ "Processing site ID #{site.id} (#{idx + 1} / #{total}) (imported_data: #{is_struct(site.imported_data)}, site_imports: #{length(site_imports)})"
)
- site_imports =
- if site.imported_data && not Enum.any?(site_imports, & &1.legacy) do
- IO.puts("Creating legacy site import entry for site ID #{site.id}")
-
- # create legacy entry if there's not one yet
- params =
- site.imported_data
- |> Imported.SiteImport.from_legacy()
- |> Map.put(:site_id, site.id)
- |> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id])
-
- legacy_site_import =
- %Imported.SiteImport{}
- |> Ecto.Changeset.change(params)
- |> insert!(dry_run?)
-
- [legacy_site_import | site_imports]
- else
- IO.puts("Legacy site import entry for site ID #{site.id} already exists")
-
- site_imports
- end
-
# adjust end date for each site import
for site_import <- site_imports do
IO.puts(
@@ -129,7 +152,7 @@ defmodule Plausible.DataMigration.SiteImports do
IO.puts("Done processing site ID #{site.id}")
end
- IO.puts("Finished")
+ IO.puts("Finished adjusting end dates of site imports.")
end
# Exposed for testing purposes
diff --git a/test/plausible/data_migration/site_imports_test.exs b/test/plausible/data_migration/site_imports_test.exs
index e1bf730f96..19a80efcf1 100644
--- a/test/plausible/data_migration/site_imports_test.exs
+++ b/test/plausible/data_migration/site_imports_test.exs
@@ -15,7 +15,7 @@ defmodule Plausible.DataMigration.SiteImportsTest do
assert :ok = SiteImports.run()
end)
- assert dry_run_output =~ "Processing 0 sites"
+ assert dry_run_output =~ "Backfilling legacy site import across 0 sites"
assert dry_run_output =~ "DRY RUN: true"
real_run_output =
@@ -23,7 +23,7 @@ defmodule Plausible.DataMigration.SiteImportsTest do
assert :ok = SiteImports.run(dry_run?: false)
end)
- assert real_run_output =~ "Processing 0 sites"
+ assert real_run_output =~ "Backfilling legacy site import across 0 sites"
assert real_run_output =~ "DRY RUN: false"
end
@@ -39,7 +39,7 @@ defmodule Plausible.DataMigration.SiteImportsTest do
assert capture_io(fn ->
assert :ok = SiteImports.run(dry_run?: false)
- end) =~ "Processing 1 sites"
+ end) =~ "Backfilling legacy site import across 1 sites"
site = Repo.reload!(site)
@@ -63,7 +63,7 @@ defmodule Plausible.DataMigration.SiteImportsTest do
assert capture_io(fn ->
assert :ok = SiteImports.run()
- end) =~ "Processing 1 sites"
+ end) =~ "Backfilling legacy site import across 1 sites"
site = Repo.reload!(site)
@@ -84,7 +84,7 @@ defmodule Plausible.DataMigration.SiteImportsTest do
assert capture_io(fn ->
assert :ok = SiteImports.run(dry_run?: false)
- end) =~ "Processing 1 sites"
+ end) =~ "Backfilling legacy site import across 1 sites"
site = Repo.reload!(site)
@@ -102,11 +102,9 @@ defmodule Plausible.DataMigration.SiteImportsTest do
|> Site.start_import(~D[2021-01-02], ~D[2020-02-02], "Google Analytics", "ok")
|> Repo.update!()
- _another_site_import = insert(:site_import, site: site)
-
assert capture_io(fn ->
assert :ok = SiteImports.run(dry_run?: false)
- end) =~ "Processing 1 site"
+ end) =~ "Backfilling legacy site import across 1 sites"
site = Repo.reload!(site)
assert [] = Imported.list_all_imports(site)
@@ -134,7 +132,7 @@ defmodule Plausible.DataMigration.SiteImportsTest do
assert capture_io(fn ->
assert :ok = SiteImports.run(dry_run?: false)
- end) =~ "Processing 1 sites"
+ end) =~ "Backfilling legacy site import across 0 sites"
site = Repo.reload!(site)
@@ -144,6 +142,69 @@ defmodule Plausible.DataMigration.SiteImportsTest do
assert site_import.end_date == ~D[2021-01-08]
assert site_import.source == :universal_analytics
end
+
+ test "only considers sites with completed site imports or 'ok' imported data" do
+ site1 =
+ insert(:site)
+ |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")
+ |> Repo.update!()
+
+ existing_import1 =
+ insert(:site_import,
+ site: site1,
+ start_date: ~D[2021-01-02],
+ end_date: ~D[2021-01-08],
+ status: :completed,
+ legacy: false
+ )
+
+ site2 =
+ insert(:site)
+ |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "ok")
+ |> Repo.update!()
+
+ existing_import2 =
+ insert(:site_import,
+ site: site2,
+ start_date: ~D[2021-01-02],
+ end_date: ~D[2021-01-08],
+ status: :failed,
+ legacy: false
+ )
+
+ site3 =
+ insert(:site)
+ |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")
+ |> Repo.update!()
+
+ site4 =
+ insert(:site)
+ |> Site.start_import(~D[2021-01-02], ~D[2021-01-08], "Google Analytics", "error")
+ |> Repo.update!()
+
+ _existing_import3 =
+ insert(:site_import,
+ site: site4,
+ start_date: ~D[2021-01-02],
+ end_date: ~D[2021-01-08],
+ status: :failed,
+ legacy: true
+ )
+
+ output =
+ capture_io(fn ->
+ assert :ok = SiteImports.run(dry_run?: false)
+ end)
+
+ assert output =~ "Backfilling legacy site import across 1 sites"
+ assert output =~ "Adjusting end dates of site imports across 2 sites"
+ assert output =~ "site ID #{site1.id} "
+ assert output =~ "site import #{existing_import1.id} "
+ assert output =~ "site ID #{site2.id} "
+ refute output =~ "site import #{existing_import2.id} "
+ refute output =~ "site ID #{site3.id} "
+ refute output =~ "site ID #{site4.id} "
+ end
end
describe "imported_stats_end_date/1" do