Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Optimize usage calculation for large accounts

Uku Taht792e534edd5b
diff --git a/lib/plausible/billing/billing.ex b/lib/plausible/billing/billing.ex
index f61919d29c..85fea0e3a7 100644
--- a/lib/plausible/billing/billing.ex
+++ b/lib/plausible/billing/billing.ex
@@ -186,14 +186,8 @@ defmodule Plausible.Billing do
end
def usage_breakdown(user) do
- sites = Plausible.Sites.owned_by(user)
-
- Enum.reduce(sites, {0, 0}, fn site, {pageviews, custom_events} ->
- usage = Plausible.Stats.Clickhouse.usage(site)
-
- {pageviews + Map.get(usage, "pageviews", 0),
- custom_events + Map.get(usage, "custom_events", 0)}
- end)
+ domains = Plausible.Sites.owned_by(user) |> Enum.map(& &1.domain)
+ Plausible.Stats.Clickhouse.usage_breakdown(domains)
end
@doc """
diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex
index ac5b2a678f..393a455087 100644
--- a/lib/plausible/stats/clickhouse.ex
+++ b/lib/plausible/stats/clickhouse.ex
@@ -174,21 +174,25 @@ defmodule Plausible.Stats.Clickhouse do
)
end
- def usage(site) do
- q = Plausible.Stats.Query.from(site.timezone, %{"period" => "30d"})
- {first_datetime, last_datetime} = utc_boundaries(q, site.timezone)
+ def usage_breakdown(domains) do
+ q = Plausible.Stats.Query.from("UTC", %{"period" => "30d"})
+ {first_datetime, last_datetime} = utc_boundaries(q, "UTC")
+
+ Enum.chunk_every(domains, 300)
+ |> Enum.reduce({0, 0}, fn domains, {pageviews_total, custom_events_total} ->
+ {chunk_pageviews, chunk_custom_events} =
+ ClickhouseRepo.one(
+ from e in "events",
+ where: e.domain in ^domains,
+ where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime,
+ select: {
+ fragment("countIf(? = 'pageview')", e.name),
+ fragment("countIf(? != 'pageview')", e.name)
+ }
+ )
- ClickhouseRepo.all(
- from e in "events",
- where: e.domain == ^site.domain,
- where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime,
- group_by: fragment("name"),
- select: {
- fragment("if(? = 'pageview', 'pageviews', 'custom_events') as name", e.name),
- fragment("count(*)")
- }
- )
- |> Enum.into(%{})
+ {pageviews_total + chunk_pageviews, custom_events_total + chunk_custom_events}
+ end)
end
def pageviews_and_visitors(site, query) do
diff --git a/test/plausible/billing/billing_test.exs b/test/plausible/billing/billing_test.exs
index b079c0b79a..998addde50 100644
--- a/test/plausible/billing/billing_test.exs
+++ b/test/plausible/billing/billing_test.exs
@@ -11,11 +11,22 @@ defmodule Plausible.BillingTest do
assert Billing.usage(user) == 0
end
- test "counts the total number of events" do
+ test "counts the total number of events from all sites the user owns" do
user = insert(:user)
- insert(:site, domain: "test-site.com", members: [user])
+ site1 = insert(:site, members: [user])
+ site2 = insert(:site, members: [user])
- assert Billing.usage(user) == 3
+ populate_stats(site1, [
+ build(:pageview),
+ build(:pageview)
+ ])
+
+ populate_stats(site2, [
+ build(:pageview),
+ build(:event, name: "custom events")
+ ])
+
+ assert Billing.usage(user) == 4
end
test "only counts usage from sites where the user is the owner" do