Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Add metrics to ingestion pipeline (#3927)
hq17523abe93ea6
diff --git a/lib/plausible/cache.ex b/lib/plausible/cache.exindex 17ec2d2cf9..c531b90c91 100644--- a/lib/plausible/cache.ex+++ b/lib/plausible/cache.ex@@ -176,26 +176,16 @@ defmodule Plausible.Cache dodefp refresh(mode, query, opts) when mode in @modes docache_name = Keyword.get(opts, :cache_name, name())- measure_duration(telemetry_event_refresh(cache_name, mode), fn ->- items = Plausible.Repo.all(query)- :ok = merge_items(items, opts)- end)+ Plausible.PromEx.Plugins.PlausibleMetrics.measure_duration(+ telemetry_event_refresh(cache_name, mode),+ fn ->+ items = Plausible.Repo.all(query)+ :ok = merge_items(items, opts)+ end+ ):okend-- defp measure_duration(event, fun) when is_function(fun, 0) do- {duration, result} = time_it(fun)- :telemetry.execute(event, %{duration: duration}, %{})- result- end-- defp time_it(fun) do- start = System.monotonic_time()- result = fun.()- stop = System.monotonic_time()- {stop - start, result}- endendendenddiff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.exindex 6dd680f615..74ed955367 100644--- a/lib/plausible/ingestion/event.ex+++ b/lib/plausible/ingestion/event.ex@@ -100,6 +100,10 @@ defmodule Plausible.Ingestion.Event do[:plausible, :ingest, :event, :dropped]end+ def telemetry_pipeline_step_duration() do+ [:plausible, :ingest, :pipeline, :step]+ end+@spec emit_telemetry_buffered(t()) :: :okdef emit_telemetry_buffered(event) do:telemetry.execute(telemetry_event_buffered(), %{}, %{@@ -119,34 +123,42 @@ defmodule Plausible.Ingestion.Event dodefp pipeline() do[- &drop_datacenter_ip/1,- &drop_shield_rule_ip/1,- &drop_shield_rule_page/1,- &put_geolocation/1,- &drop_shield_rule_country/1,- &put_user_agent/1,- &put_basic_info/1,- &put_referrer/1,- &put_utm_tags/1,- &put_props/1,- &put_revenue/1,- &put_salts/1,- &put_user_id/1,- &validate_clickhouse_event/1,- ®ister_session/1,- &write_to_buffer/1+ drop_datacenter_ip: &drop_datacenter_ip/1,+ drop_shield_rule_page: &drop_shield_rule_page/1,+ drop_shield_rule_ip: &drop_shield_rule_ip/1,+ put_geolocation: &put_geolocation/1,+ drop_shield_rule_country: &drop_shield_rule_country/1,+ put_user_agent: &put_user_agent/1,+ put_basic_info: &put_basic_info/1,+ put_referrer: &put_referrer/1,+ put_utm_tags: &put_utm_tags/1,+ put_props: &put_props/1,+ put_revenue: &put_revenue/1,+ put_salts: &put_salts/1,+ put_user_id: &put_user_id/1,+ validate_clickhouse_event: &validate_clickhouse_event/1,+ register_session: ®ister_session/1,+ write_to_buffer: &write_to_buffer/1]enddefp process_unless_dropped(%__MODULE__{} = initial_event, pipeline) do- Enum.reduce_while(pipeline, initial_event, fn pipeline_step, acc_event ->- case pipeline_step.(acc_event) do- %__MODULE__{dropped?: true} = dropped -> {:halt, dropped}- %__MODULE__{dropped?: false} = event -> {:cont, event}- end+ Enum.reduce_while(pipeline, initial_event, fn {step_name, step_fn}, acc_event ->+ Plausible.PromEx.Plugins.PlausibleMetrics.measure_duration(+ telemetry_pipeline_step_duration(),+ fn -> execute_step(step_fn, acc_event) end,+ %{step: "#{step_name}"}+ )end)end+ defp execute_step(step_fn, acc_event) do+ case step_fn.(acc_event) do+ %__MODULE__{dropped?: true} = dropped -> {:halt, dropped}+ %__MODULE__{dropped?: false} = event -> {:cont, event}+ end+ end+defp new(domain, request) dostruct!(__MODULE__, domain: domain, request: request)enddiff --git a/lib/plausible/telemetry/plausible_metrics.ex b/lib/plausible/telemetry/plausible_metrics.exindex 80e78d25b5..17e24e3371 100644--- a/lib/plausible/telemetry/plausible_metrics.ex+++ b/lib/plausible/telemetry/plausible_metrics.ex@@ -47,6 +47,16 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics dounit: {:native, :millisecond},measurement: :duration),+ distribution(+ metric_prefix ++ [:ingest, :events, :pipeline, :steps],+ event_name: Ingestion.Event.telemetry_pipeline_step_duration(),+ reporter_options: [+ buckets: [10, 50, 100, 250, 350, 500, 1000, 5000, 10_000, 100_000, 500_000]+ ],+ unit: {:native, :microsecond},+ measurement: :duration,+ tags: [:step]+ ),counter(metric_prefix ++ [:ingest, :events, :buffered, :total],event_name: Ingestion.Event.telemetry_event_buffered()@@ -110,6 +120,19 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do})end+ def measure_duration(event, fun, meta \\ %{}) when is_function(fun, 0) do+ {duration, result} = time_it(fun)+ :telemetry.execute(event, %{duration: duration}, meta)+ result+ end++ defp time_it(fun) do+ start = System.monotonic_time()+ result = fun.()+ stop = System.monotonic_time()+ {stop - start, result}+ end+defp write_buffer_metrics(metric_prefix, poll_rate) doPolling.build(:write_buffer_metrics,