Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Allow Site DB lookups during ingestion phase (#2408)

Adam Rutkowski101e5a68b5ff
diff --git a/config/.env.dev b/config/.env.dev
index d1d896e203..d7e46e3854 100644
--- a/config/.env.dev
+++ b/config/.env.dev
@@ -14,3 +14,5 @@ PADDLE_VENDOR_ID=3942
GOOGLE_CLIENT_ID=875387135161-l8tp53dpt7fdhdg9m1pc3vl42si95rh0.apps.googleusercontent.com
GOOGLE_CLIENT_SECRET=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
+
+PROMEX_DISABLED=false
diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex
index 2a70623356..577330e173 100644
--- a/lib/plausible/ingestion/event.ex
+++ b/lib/plausible/ingestion/event.ex
@@ -20,6 +20,7 @@ defmodule Plausible.Ingestion.Event do
events when is_list(events) <- map_domains(event, request),
events when is_list(events) <- put_user_id(events, request, salts),
{:ok, events} <- validate_events(events),
+ :ok <- measured_sites_get(events),
events when is_list(events) <- register_session(events, request, salts) do
Enum.each(events, &Plausible.Event.WriteBuffer.insert/1)
end
@@ -332,4 +333,27 @@ defmodule Plausible.Ingestion.Event do
nil
end
end
+
+ defp measured_sites_get(events) do
+ events
+ |> Enum.map(& &1.domain)
+ |> Enum.uniq()
+ |> Enum.each(fn domain ->
+ if should_lookup_site?(domain) do
+ start = System.monotonic_time()
+ Plausible.Sites.get_by_domain(domain)
+ stop = System.monotonic_time()
+
+ :telemetry.execute(
+ [:plausible, :ingestion, :site, :lookup],
+ %{duration: stop - start},
+ %{}
+ )
+ end
+ end)
+ end
+
+ defp should_lookup_site?(domain) do
+ FunWithFlags.enabled?(:ingestion_pg_lookup, for: domain)
+ end
end
diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.ex
index 5b7aebd8e7..c9c663addf 100644
--- a/lib/plausible/sites.ex
+++ b/lib/plausible/sites.ex
@@ -4,6 +4,10 @@ defmodule Plausible.Sites do
alias Plausible.Site.SharedLink
import Ecto.Query
+ def get_by_domain(domain) do
+ Repo.get_by(Site, domain: domain)
+ end
+
def create(user, params) do
limit = Plausible.Billing.sites_limit(user)
diff --git a/lib/plausible/telemetry/plausible_metrics.ex b/lib/plausible/telemetry/plausible_metrics.ex
index 3f1aa8b700..4f1d5dfd87 100644
--- a/lib/plausible/telemetry/plausible_metrics.ex
+++ b/lib/plausible/telemetry/plausible_metrics.ex
@@ -18,6 +18,33 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do
]
end
+ @impl true
+ def event_metrics(_opts) do
+ metric_prefix = [:plausible, :profiling]
+
+ Event.build(
+ :plausible_event_metrics,
+ [
+ counter(
+ metric_prefix ++ [:ingestion, :site, :lookup, :counter],
+ event_name: [:plausible, :ingestion, :site, :lookup],
+ measurement: fn _ -> 1 end,
+ description: "Ingestion site lookup counter"
+ ),
+ distribution(
+ metric_prefix ++ [:ingestion, :site, :lookup, :duration, :milliseconds],
+ event_name: [:plausible, :ingestion, :site, :lookup],
+ description: "Ingestion site lookup duration",
+ measurement: :duration,
+ reporter_options: [
+ buckets: [5, 10, 50, 250, 1_000, 5_000]
+ ],
+ unit: {:native, :millisecond}
+ )
+ ]
+ )
+ end
+
@doc """
Add telemetry events for Session and Event write buffers
"""
@@ -51,13 +78,15 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do
def execute_cache_metrics do
user_agents_count =
case Cachex.stats(:user_agents) do
- {:ok, stats} -> stats
+ # https://github.com/whitfin/cachex/pull/301
+ {:ok, %{writes: w, evictions: e}} when is_integer(w) and is_integer(e) -> w - e
_ -> 0
end
sessions_count =
case Cachex.stats(:sessions) do
- {:ok, stats} -> stats
+ # https://github.com/whitfin/cachex/pull/301
+ {:ok, %{writes: w, evictions: e}} when is_integer(w) and is_integer(e) -> w - e
_ -> 0
end