Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Limit resource usage in teams backfill data migration (#4708)

Adrian Gruntkowski3023d322d4cc
diff --git a/lib/plausible/data_migration/backfill_teams.ex b/lib/plausible/data_migration/backfill_teams.ex
index 9e544f1017..db03c7b0d7 100644
--- a/lib/plausible/data_migration/backfill_teams.ex
+++ b/lib/plausible/data_migration/backfill_teams.ex
@@ -9,6 +9,7 @@ defmodule Plausible.DataMigration.BackfillTeams do
alias Plausible.Teams
@repo Plausible.DataMigration.PostgresRepo
+ @max_concurrency 12
defmacrop is_distinct(f1, f2) do
quote do
@@ -24,7 +25,7 @@ defmodule Plausible.DataMigration.BackfillTeams do
Application.get_env(:plausible, Plausible.Repo)[:url]
)
- @repo.start(db_url, pool_size: System.schedulers_online() * 2)
+ @repo.start(db_url, pool_size: 2 * @max_concurrency)
backfill()
end
@@ -425,7 +426,8 @@ defmodule Plausible.DataMigration.BackfillTeams do
set: [team_id: team.id]
)
end,
- timeout: :infinity
+ timeout: :infinity,
+ max_concurrency: @max_concurrency
)
if rem(idx, 10) == 0 do
@@ -465,7 +467,8 @@ defmodule Plausible.DataMigration.BackfillTeams do
|> Ecto.Changeset.put_change(:updated_at, owner.updated_at)
|> @repo.insert!()
end,
- timeout: :infinity
+ timeout: :infinity,
+ max_concurrency: @max_concurrency
)
if rem(idx, 10) == 0 do
@@ -511,7 +514,8 @@ defmodule Plausible.DataMigration.BackfillTeams do
IO.write(".")
end
end,
- timeout: :infinity
+ timeout: :infinity,
+ max_concurrency: @max_concurrency
)
|> Stream.run()
end
@@ -532,7 +536,8 @@ defmodule Plausible.DataMigration.BackfillTeams do
IO.write(".")
end
end,
- timeout: :infinity
+ timeout: :infinity,
+ max_concurrency: @max_concurrency
)
|> Stream.run()
end
@@ -598,7 +603,8 @@ defmodule Plausible.DataMigration.BackfillTeams do
IO.write(".")
end
end,
- timeout: :infinity
+ timeout: :infinity,
+ max_concurrency: @max_concurrency
)
|> Stream.run()
end