Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Implement teams consistency check and improve teams backfill (#4733)

Adrian Gruntkowski5dbe1870e040
diff --git a/lib/plausible/data_migration/backfill_teams.ex b/lib/plausible/data_migration/backfill_teams.ex
index 89314e57a6..70cf839234 100644
--- a/lib/plausible/data_migration/backfill_teams.ex
+++ b/lib/plausible/data_migration/backfill_teams.ex
@@ -88,10 +88,17 @@ defmodule Plausible.DataMigration.BackfillTeams do
is_distinct(o.trial_expiry_date, t.trial_expiry_date) or
is_distinct(o.accept_traffic_until, t.accept_traffic_until) or
is_distinct(o.allow_next_upgrade_override, t.allow_next_upgrade_override) or
- is_distinct(o.grace_period["id"], t.grace_period["id"]) or
- is_distinct(o.grace_period["is_over"], t.grace_period["is_over"]) or
- is_distinct(o.grace_period["end_date"], t.grace_period["end_date"]) or
- is_distinct(o.grace_period["manual_lock"], t.grace_period["manual_lock"]),
+ (is_distinct(o.grace_period, t.grace_period) and
+ (is_distinct(o.grace_period["id"], t.grace_period["id"]) or
+ (is_nil(o.grace_period["is_over"]) and t.grace_period["is_over"] == true) or
+ (o.grace_period["is_over"] == true and t.grace_period["is_over"] == false) or
+ (o.grace_period["is_over"] == false and t.grace_period["is_over"] == true) or
+ is_distinct(o.grace_period["end_date"], t.grace_period["end_date"]) or
+ (is_nil(o.grace_period["manual_lock"]) and t.grace_period["manual_lock"] == true) or
+ (o.grace_period["manual_lock"] == true and
+ t.grace_period["manual_lock"] == false) or
+ (o.grace_period["manual_lock"] == false and
+ t.grace_period["manual_lock"] == true))),
preload: [team_memberships: {tm, user: o}]
)
|> @repo.all(timeout: :infinity)
@@ -482,11 +489,17 @@ defmodule Plausible.DataMigration.BackfillTeams do
:allow_next_upgrade_override,
owner.allow_next_upgrade_override
)
- |> Ecto.Changeset.put_embed(:grace_period, owner.grace_period)
+ |> Ecto.Changeset.put_embed(:grace_period, embed_params(owner.grace_period))
|> @repo.update!()
end)
end
+ defp embed_params(nil), do: nil
+
+ defp embed_params(grace_period) do
+ Map.from_struct(grace_period)
+ end
+
defp backfill_subscriptions(subscriptions) do
subscriptions
|> Enum.with_index()
diff --git a/lib/plausible/data_migration/teams_consistency_check.ex b/lib/plausible/data_migration/teams_consistency_check.ex
new file mode 100644
index 0000000000..118a4a238d
--- /dev/null
+++ b/lib/plausible/data_migration/teams_consistency_check.ex
@@ -0,0 +1,236 @@
+defmodule Plausible.DataMigration.TeamsConsitencyCheck do
+ @moduledoc """
+ Verify consistency of teams.
+ """
+
+ import Ecto.Query
+
+ alias Plausible.Teams
+
+ @repo Plausible.DataMigration.PostgresRepo
+
+ defmacrop is_distinct(f1, f2) do
+ quote do
+ fragment("? IS DISTINCT FROM ?", unquote(f1), unquote(f2))
+ end
+ end
+
+ def run() do
+ # Teams consistency check
+ db_url =
+ System.get_env(
+ "TEAMS_MIGRATION_DB_URL",
+ Application.get_env(:plausible, Plausible.Repo)[:url]
+ )
+
+ @repo.start(db_url, pool_size: 1)
+
+ check()
+ end
+
+ defp check() do
+ # Sites without teams
+
+ sites_without_teams_count =
+ from(
+ s in Plausible.Site,
+ where: is_nil(s.team_id)
+ )
+ |> @repo.aggregate(:count, timeout: :infinity)
+
+ log("#{sites_without_teams_count} sites without teams")
+
+ # Teams without owner
+
+ owner_membership_query =
+ from(
+ tm in Teams.Membership,
+ where: tm.team_id == parent_as(:team).id,
+ where: tm.role == :owner,
+ select: 1
+ )
+
+ teams_without_owner_count =
+ from(
+ t in Plausible.Teams.Team,
+ as: :team,
+ where: not exists(owner_membership_query)
+ )
+ |> @repo.aggregate(:count, timeout: :infinity)
+
+ log("#{teams_without_owner_count} teams without owner")
+
+ # Subscriptions without teams
+
+ subscriptions_without_teams_count =
+ from(
+ s in Plausible.Billing.Subscription,
+ where: is_nil(s.team_id)
+ )
+ |> @repo.aggregate(:count, timeout: :infinity)
+
+ log("#{subscriptions_without_teams_count} subscriptions without teams")
+
+ # Subscriptions out of sync
+
+ subscriptions_out_of_sync_count =
+ from(
+ s in Plausible.Billing.Subscription,
+ inner_join: u in assoc(s, :user),
+ left_join: tm in assoc(u, :team_memberships),
+ on: tm.role == :owner,
+ where: s.team_id != tm.team_id
+ )
+ |> @repo.aggregate(:count, timeout: :infinity)
+
+ log("#{subscriptions_out_of_sync_count} subscriptions out of sync")
+
+ # Enterprise plans without teams
+
+ enterprise_plans_without_teams_count =
+ from(
+ ep in Plausible.Billing.EnterprisePlan,
+ where: is_nil(ep.team_id)
+ )
+ |> @repo.aggregate(:count, timeout: :infinity)
+
+ log("#{enterprise_plans_without_teams_count} enterprise_plans without teams")
+
+ # Enterprise plans out of sync
+
+ enterprise_plans_out_of_sync_count =
+ from(
+ ep in Plausible.Billing.EnterprisePlan,
+ inner_join: u in assoc(ep, :user),
+ left_join: tm in assoc(u, :team_memberships),
+ on: tm.role == :owner,
+ where: ep.team_id != tm.team_id
+ )
+ |> @repo.aggregate(:count, timeout: :infinity)
+
+ log("#{enterprise_plans_out_of_sync_count} enterprise_plans out of sync")
+
+ # Teams out of sync
+
+ teams_out_of_sync_count =
+ from(
+ t in Teams.Team,
+ inner_join: tm in assoc(t, :team_memberships),
+ inner_join: o in assoc(tm, :user),
+ where: tm.role == :owner,
+ where:
+ is_distinct(o.trial_expiry_date, t.trial_expiry_date) or
+ is_distinct(o.accept_traffic_until, t.accept_traffic_until) or
+ is_distinct(o.allow_next_upgrade_override, t.allow_next_upgrade_override) or
+ (is_distinct(o.grace_period, t.grace_period) and
+ (is_distinct(o.grace_period["id"], t.grace_period["id"]) or
+ (is_nil(o.grace_period["is_over"]) and t.grace_period["is_over"] == true) or
+ (o.grace_period["is_over"] == true and t.grace_period["is_over"] == false) or
+ (o.grace_period["is_over"] == false and t.grace_period["is_over"] == true) or
+ is_distinct(o.grace_period["end_date"], t.grace_period["end_date"]) or
+ (is_nil(o.grace_period["manual_lock"]) and t.grace_period["manual_lock"] == true) or
+ (o.grace_period["manual_lock"] == true and
+ t.grace_period["manual_lock"] == false) or
+ (o.grace_period["manual_lock"] == false and
+ t.grace_period["manual_lock"] == true))),
+ preload: [team_memberships: {tm, user: o}]
+ )
+ |> @repo.aggregate(:count, timeout: :infinity)
+
+ log("#{teams_out_of_sync_count} teams out of sync")
+
+ # Non-owner site memberships out of sync
+
+ respective_guest_memberships_query =
+ from(
+ tm in Teams.Membership,
+ inner_join: gm in assoc(tm, :guest_memberships),
+ on:
+ gm.site_id == parent_as(:site_membership).site_id and
+ ((gm.role == :viewer and parent_as(:site_membership).role == :viewer) or
+ (gm.role == :editor and parent_as(:site_membership).role == :admin)),
+ where: tm.user_id == parent_as(:site_membership).user_id,
+ select: 1
+ )
+
+ out_of_sync_nonowner_memberships_count =
+ from(
+ m in Plausible.Site.Membership,
+ as: :site_membership,
+ where: m.role != :owner,
+ where: not exists(respective_guest_memberships_query)
+ )
+ |> @repo.aggregate(:count, timeout: :infinity)
+
+ log("#{out_of_sync_nonowner_memberships_count} out of sync non-owner site memberships")
+
+ # Owner site memberships out of sync
+
+ respective_owner_memberships_query =
+ from(
+ tm in Teams.Membership,
+ where: tm.team_id == parent_as(:site).team_id and tm.role == :owner,
+ select: 1
+ )
+
+ out_of_sync_owner_memberships_count =
+ from(
+ m in Plausible.Site.Membership,
+ as: :site_membership,
+ inner_join: s in assoc(m, :site),
+ as: :site,
+ where: m.role == :owner,
+ where: not exists(respective_owner_memberships_query)
+ )
+ |> @repo.aggregate(:count, timeout: :infinity)
+
+ log("#{out_of_sync_owner_memberships_count} out of sync owner site memberships")
+
+ # Site invitations out of sync
+
+ respective_guest_invitations_query =
+ from(
+ gi in Teams.GuestInvitation,
+ inner_join: ti in assoc(gi, :team_invitation),
+ on: ti.email == parent_as(:site_invitation).email,
+ where: gi.site_id == parent_as(:site_invitation).site_id,
+ select: 1
+ )
+
+ out_of_sync_site_invitations_count =
+ from(
+ i in Plausible.Auth.Invitation,
+ as: :site_invitation,
+ where: i.role != :owner,
+ where: not exists(respective_guest_invitations_query)
+ )
+ |> @repo.aggregate(:count, timeout: :infinity)
+
+ log("#{out_of_sync_site_invitations_count} out of sync site invitations")
+
+ # Site invitations out of sync
+
+ respective_site_transfers_query =
+ from(
+ st in Teams.SiteTransfer,
+ where: st.email == parent_as(:site_invitation).email,
+ where: st.site_id == parent_as(:site_invitation).site_id,
+ select: 1
+ )
+
+ out_of_sync_site_transfers_count =
+ from(
+ i in Plausible.Auth.Invitation,
+ as: :site_invitation,
+ where: i.role == :owner,
+ where: not exists(respective_site_transfers_query)
+ )
+ |> @repo.aggregate(:count, timeout: :infinity)
+
+ log("#{out_of_sync_site_transfers_count} out of sync site transfers")
+ end
+
+ defp log(msg) do
+ IO.puts("[#{NaiveDateTime.utc_now(:second)}] #{msg}")
+ end
+end
diff --git a/lib/plausible/teams/team.ex b/lib/plausible/teams/team.ex
index e1fe3a056d..357e268eeb 100644
--- a/lib/plausible/teams/team.ex
+++ b/lib/plausible/teams/team.ex
@@ -35,7 +35,7 @@ defmodule Plausible.Teams.Team do
|> put_change(:trial_expiry_date, user.trial_expiry_date)
|> put_change(:accept_traffic_until, user.accept_traffic_until)
|> put_change(:allow_next_upgrade_override, user.allow_next_upgrade_override)
- |> put_embed(:grace_period, user.grace_period)
+ |> put_embed(:grace_period, embed_params(user.grace_period))
|> put_change(:inserted_at, user.inserted_at)
|> put_change(:updated_at, user.updated_at)
end
@@ -63,6 +63,12 @@ defmodule Plausible.Teams.Team do
)
end
+ defp embed_params(nil), do: nil
+
+ defp embed_params(grace_period) do
+ Map.from_struct(grace_period)
+ end
+
defp trial_expiry() do
on_ee do
Date.utc_today() |> Date.shift(day: 30)