Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Optimize usage calculation for large accounts
Uku Taht792e534edd5b
diff --git a/lib/plausible/billing/billing.ex b/lib/plausible/billing/billing.exindex f61919d29c..85fea0e3a7 100644--- a/lib/plausible/billing/billing.ex+++ b/lib/plausible/billing/billing.ex@@ -186,14 +186,8 @@ defmodule Plausible.Billing doenddef usage_breakdown(user) do- sites = Plausible.Sites.owned_by(user)-- Enum.reduce(sites, {0, 0}, fn site, {pageviews, custom_events} ->- usage = Plausible.Stats.Clickhouse.usage(site)-- {pageviews + Map.get(usage, "pageviews", 0),- custom_events + Map.get(usage, "custom_events", 0)}- end)+ domains = Plausible.Sites.owned_by(user) |> Enum.map(& &1.domain)+ Plausible.Stats.Clickhouse.usage_breakdown(domains)end@doc """diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex ac5b2a678f..393a455087 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -174,21 +174,25 @@ defmodule Plausible.Stats.Clickhouse do)end- def usage(site) do- q = Plausible.Stats.Query.from(site.timezone, %{"period" => "30d"})- {first_datetime, last_datetime} = utc_boundaries(q, site.timezone)+ def usage_breakdown(domains) do+ q = Plausible.Stats.Query.from("UTC", %{"period" => "30d"})+ {first_datetime, last_datetime} = utc_boundaries(q, "UTC")++ Enum.chunk_every(domains, 300)+ |> Enum.reduce({0, 0}, fn domains, {pageviews_total, custom_events_total} ->+ {chunk_pageviews, chunk_custom_events} =+ ClickhouseRepo.one(+ from e in "events",+ where: e.domain in ^domains,+ where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime,+ select: {+ fragment("countIf(? = 'pageview')", e.name),+ fragment("countIf(? != 'pageview')", e.name)+ }+ )- ClickhouseRepo.all(- from e in "events",- where: e.domain == ^site.domain,- where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime,- group_by: fragment("name"),- select: {- fragment("if(? = 'pageview', 'pageviews', 'custom_events') as name", e.name),- fragment("count(*)")- }- )- |> Enum.into(%{})+ {pageviews_total + chunk_pageviews, custom_events_total + chunk_custom_events}+ end)enddef pageviews_and_visitors(site, query) dodiff --git a/test/plausible/billing/billing_test.exs b/test/plausible/billing/billing_test.exsindex b079c0b79a..998addde50 100644--- a/test/plausible/billing/billing_test.exs+++ b/test/plausible/billing/billing_test.exs@@ -11,11 +11,22 @@ defmodule Plausible.BillingTest doassert Billing.usage(user) == 0end- test "counts the total number of events" do+ test "counts the total number of events from all sites the user owns" douser = insert(:user)- insert(:site, domain: "test-site.com", members: [user])+ site1 = insert(:site, members: [user])+ site2 = insert(:site, members: [user])- assert Billing.usage(user) == 3+ populate_stats(site1, [+ build(:pageview),+ build(:pageview)+ ])++ populate_stats(site2, [+ build(:pageview),+ build(:event, name: "custom events")+ ])++ assert Billing.usage(user) == 4endtest "only counts usage from sites where the user is the owner" do