Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Populate log_comment with debug information, /debug/clickhouse route (#4435)
Karl-Aksel Puulmann496796027822
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex bc1bb3c11f..83cbe40a4f 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -14,6 +14,8 @@ All notable changes to this project will be documented in this file.- Add `does_not_contain` filter support to dashboard- Traffic drop notifications plausible/analytics#4300- Add search and pagination functionality into Google Keywords > Details modal+- ClickHouse system.query_log table log_comment column now contains information about source of queries. Useful for debugging+- New /debug/clickhouse route for super admins which shows information on clickhouse queries executed by user### Removed- Deprecate `ECTO_IPV6` and `ECTO_CH_IPV6` env vars in CE plausible/analytics#4245diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.exindex 37d5961cb9..35821d8d46 100644--- a/lib/plausible/clickhouse_repo.ex+++ b/lib/plausible/clickhouse_repo.ex@@ -30,4 +30,17 @@ defmodule Plausible.ClickhouseRepo do|> Enum.to_list()|> Keyword.values()end++ @impl true+ def prepare_query(_operation, query, opts) do+ {plausible_query, opts} = Keyword.pop(opts, :query)+ log_comment = if(plausible_query, do: Jason.encode!(plausible_query.debug_metadata), else: "")++ opts =+ Keyword.update(opts, :settings, [log_comment: log_comment], fn settings ->+ [{:log_comment, log_comment} | settings]+ end)++ {query, opts}+ endenddiff --git a/lib/plausible/data_migration/locations_sync.ex b/lib/plausible/data_migration/locations_sync.exindex 9b81179c84..7ebba6098b 100644--- a/lib/plausible/data_migration/locations_sync.ex+++ b/lib/plausible/data_migration/locations_sync.ex@@ -88,7 +88,7 @@ defmodule Plausible.DataMigration.LocationsSync doenddef run() do- cluster? = Plausible.MigrationUtils.clustered_table?("sessions_v2")+ cluster? = Plausible.IngestRepo.clustered_table?("sessions_v2"){:ok, _} = run_sql("truncate-location-data-table", cluster?: cluster?){:ok, _} = run_sql("create-location-data-table", cluster?: cluster?)diff --git a/lib/plausible/data_migration/populate_event_session_columns.ex b/lib/plausible/data_migration/populate_event_session_columns.exindex 796fc0435d..9ae543a993 100644--- a/lib/plausible/data_migration/populate_event_session_columns.ex+++ b/lib/plausible/data_migration/populate_event_session_columns.ex@@ -22,7 +22,7 @@ defmodule Plausible.DataMigration.PopulateEventSessionColumns do}def run(opts \\ []) do- cluster? = Plausible.MigrationUtils.clustered_table?("sessions_v2")+ cluster? = Plausible.IngestRepo.clustered_table?("sessions_v2"){:ok, _} =run_sql("create-sessions-dictionary",@@ -54,7 +54,7 @@ defmodule Plausible.DataMigration.PopulateEventSessionColumns doenddef kill(opts \\ []) do- cluster? = Plausible.MigrationUtils.clustered_table?("events_v2")+ cluster? = Plausible.IngestRepo.clustered_table?("events_v2")report_progress(opts)diff --git a/lib/plausible/data_migration/versioned_sessions.ex b/lib/plausible/data_migration/versioned_sessions.exindex 7152d2d23b..4288683c9d 100644--- a/lib/plausible/data_migration/versioned_sessions.ex+++ b/lib/plausible/data_migration/versioned_sessions.ex@@ -21,7 +21,7 @@ defmodule Plausible.DataMigration.VersionedSessions dounique_suffix = Timex.now() |> Calendar.strftime(@suffix_format)- cluster? = Plausible.MigrationUtils.clustered_table?("sessions_v2")+ cluster? = Plausible.IngestRepo.clustered_table?("sessions_v2"){:ok, %{rows: partitions}} = run_sql("list-partitions")partitions = Enum.map(partitions, fn [part] -> part end)diff --git a/lib/plausible/ingest_repo.ex b/lib/plausible/ingest_repo.exindex 3fe099f10f..703082e1ce 100644--- a/lib/plausible/ingest_repo.ex+++ b/lib/plausible/ingest_repo.ex@@ -14,4 +14,11 @@ defmodule Plausible.IngestRepo doimport Ecto.Query, only: [from: 1, from: 2]endend++ def clustered_table?(table) do+ case query("SELECT 1 FROM system.replicas WHERE table = '#{table}'") do+ {:ok, %{rows: []}} -> false+ {:ok, _} -> true+ end+ endenddiff --git a/lib/plausible/migration_utils.ex b/lib/plausible/migration_utils.exindex a5f4eba6bf..e477537e76 100644--- a/lib/plausible/migration_utils.ex+++ b/lib/plausible/migration_utils.ex@@ -3,20 +3,15 @@ defmodule Plausible.MigrationUtils doBase module for to use in Clickhouse migrations"""- def on_cluster_statement(table) do- if(clustered_table?(table), do: "ON CLUSTER '{cluster}'", else: "")- end+ alias Plausible.IngestRepo- def clustered_table?(table) do- case Plausible.IngestRepo.query("SELECT 1 FROM system.replicas WHERE table = '#{table}'") do- {:ok, %{rows: []}} -> false- {:ok, _} -> true- end+ def on_cluster_statement(table) do+ if(IngestRepo.clustered_table?(table), do: "ON CLUSTER '{cluster}'", else: "")end# See https://clickhouse.com/docs/en/sql-reference/dictionaries#clickhouse for contextdef dictionary_connection_params() do- Plausible.IngestRepo.config()+ IngestRepo.config()|> Enum.map(fn{:database, database} -> "DB '#{database}'"{:username, username} -> "USER '#{username}'"diff --git a/lib/plausible/stats.ex b/lib/plausible/stats.exindex 2db853b4dd..0baa6b9c07 100644--- a/lib/plausible/stats.ex+++ b/lib/plausible/stats.ex@@ -22,7 +22,7 @@ defmodule Plausible.Stats dooptimized_query|> SQL.QueryBuilder.build(site)- |> ClickhouseRepo.all()+ |> ClickhouseRepo.all(query: query)|> QueryResult.from(optimized_query)enddiff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex 790b5ed719..bcf1faffa8 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -33,7 +33,7 @@ defmodule Plausible.Stats.Aggregate doendPlausible.ClickhouseRepo.parallel_tasks([- run_query_task(q),+ run_query_task(q, query),time_on_page_task])|> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end)@@ -44,8 +44,8 @@ defmodule Plausible.Stats.Aggregate do|> Enum.into(%{})end- defp run_query_task(nil), do: fn -> %{} end- defp run_query_task(q), do: fn -> ClickhouseRepo.one(q) end+ defp run_query_task(nil, _query), do: fn -> %{} end+ defp run_query_task(q, query), do: fn -> ClickhouseRepo.one(q, query: query) enddefp aggregate_time_on_page(site, query) dowindowed_pages_q =@@ -87,7 +87,7 @@ defmodule Plausible.Stats.Aggregate dofrom e in Ecto.Query.subquery(avg_time_per_page_transition_q),select: fragment("avg(ifNotFinite(?,NULL))", e.avg)- %{time_on_page: ClickhouseRepo.one(time_on_page_q)}+ %{time_on_page: ClickhouseRepo.one(time_on_page_q, query: query)}end@metrics_to_round [:bounce_rate, :time_on_page, :visit_duration, :sample_percent]diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 05c150a076..fa6c576185 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -33,7 +33,7 @@ defmodule Plausible.Stats.Breakdown doq|> apply_pagination(pagination)- |> ClickhouseRepo.all()+ |> ClickhouseRepo.all(query: query)|> QueryResult.from(query_with_metrics)|> build_breakdown_result(query_with_metrics, metrics)|> maybe_add_time_on_page(site, query_with_metrics, metrics)@@ -150,7 +150,7 @@ defmodule Plausible.Stats.Breakdown doendtimed_pages_q- |> Plausible.ClickhouseRepo.all()+ |> Plausible.ClickhouseRepo.all(query: query)|> Map.new()enddiff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.exindex 02d02c9f29..3abd8194c5 100644--- a/lib/plausible/stats/legacy/legacy_query_builder.ex+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex@@ -5,12 +5,12 @@ defmodule Plausible.Stats.Legacy.QueryBuilder doalias Plausible.Stats.{Filters, Interval, Query}- def from(site, params) do+ def from(site, params, debug_metadata) donow = NaiveDateTime.utc_now(:second)query =Query- |> struct!(now: now, timezone: site.timezone)+ |> struct!(now: now, timezone: site.timezone, debug_metadata: debug_metadata)|> put_period(site, params)|> put_dimensions(params)|> put_interval(params)diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex 7de4e931b4..5bfcbf7c99 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -21,20 +21,21 @@ defmodule Plausible.Stats.Query doinclude: %{imports: false,time_labels: false- }+ },+ debug_metadata: %{}require OpenTelemetry.Tracer, as: Traceralias Plausible.Stats.{Filters, Imported, Legacy}@type t :: %__MODULE__{}- def build(site, params) do+ def build(site, params, debug_metadata) dowith {:ok, query_data} <- Filters.QueryParser.parse(site, params) doquery =struct!(__MODULE__, Map.to_list(query_data))|> put_imported_opts(site, %{})|> put_experimental_reduced_joins(site, params)- |> struct!(v2: true, now: NaiveDateTime.utc_now(:second))+ |> struct!(v2: true, now: NaiveDateTime.utc_now(:second), debug_metadata: debug_metadata){:ok, query}end@@ -43,9 +44,8 @@ defmodule Plausible.Stats.Query do@doc """Builds query from old-style params. New code should prefer Query.build"""- @spec from(Plausible.Site.t(), map()) :: t()- def from(site, params) do- Legacy.QueryBuilder.from(site, params)+ def from(site, params, debug_metadata \\ %{}) do+ Legacy.QueryBuilder.from(site, params, debug_metadata)enddef put_experimental_reduced_joins(query, site, params) dodiff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex 513debc597..6971fe7dcf 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -39,7 +39,7 @@ defmodule Plausible.Stats.Timeseries doq = SQL.QueryBuilder.build(query_with_metrics, site)q- |> ClickhouseRepo.all()+ |> ClickhouseRepo.all(query: query)|> QueryResult.from(query_with_metrics)|> build_timeseries_result(query_with_metrics, currency)|> transform_keys(%{group_conversion_rate: :conversion_rate})diff --git a/lib/plausible_web/controllers/api/external_query_api_controller.ex b/lib/plausible_web/controllers/api/external_query_api_controller.exindex d2bf01401f..5b88d39e7c 100644--- a/lib/plausible_web/controllers/api/external_query_api_controller.ex+++ b/lib/plausible_web/controllers/api/external_query_api_controller.ex@@ -9,7 +9,7 @@ defmodule PlausibleWeb.Api.ExternalQueryApiController dodef query(conn, params) dosite = Repo.preload(conn.assigns.site, :owner)- case Query.build(site, params) do+ case Query.build(site, params, debug_metadata(conn)) do{:ok, query} ->results = Plausible.Stats.query(site, query)json(conn, results)diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.exindex 1ba1a8079d..fa3188550d 100644--- a/lib/plausible_web/controllers/api/external_stats_controller.ex+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex@@ -16,7 +16,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController dowith :ok <- validate_period(params),:ok <- validate_date(params),- query <- Query.from(site, params),+ query <- Query.from(site, params, debug_metadata(conn)),:ok <- validate_filters(site, query.filters),{:ok, metrics} <- parse_and_validate_metrics(params, query),:ok <- ensure_custom_props_access(site, query) do@@ -55,7 +55,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController dowith :ok <- validate_period(params),:ok <- validate_date(params),:ok <- validate_property(params),- query <- Query.from(site, params),+ query <- Query.from(site, params, debug_metadata(conn)),:ok <- validate_filters(site, query.filters),{:ok, metrics} <- parse_and_validate_metrics(params, query),{:ok, limit} <- validate_or_default_limit(params),@@ -262,7 +262,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController dowith :ok <- validate_period(params),:ok <- validate_date(params),:ok <- validate_interval(params),- query <- Query.from(site, params),+ query <- Query.from(site, params, debug_metadata(conn)),:ok <- validate_filters(site, query.filters),{:ok, metrics} <- parse_and_validate_metrics(params, query),:ok <- ensure_custom_props_access(site, query) dodiff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 2114a973d1..8204abda2d 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -104,7 +104,7 @@ defmodule PlausibleWeb.Api.StatsController dowith {:ok, dates} <- parse_date_params(params),:ok <- validate_interval(params),:ok <- validate_interval_granularity(site, params, dates),- query = Query.from(site, params),+ query = Query.from(site, params, debug_metadata(conn)),{:ok, metric} <- parse_and_validate_graph_metric(params, query) dotimeseries_query =if query.period == "realtime" do@@ -206,7 +206,7 @@ defmodule PlausibleWeb.Api.StatsController dodef top_stats(conn, params) dosite = conn.assigns[:site]- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))comparison_opts = parse_comparison_opts(params)@@ -501,7 +501,7 @@ defmodule PlausibleWeb.Api.StatsController dodef sources(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "property", "visit:source")- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))pagination = parse_pagination(params)extra_metrics =@@ -534,7 +534,7 @@ defmodule PlausibleWeb.Api.StatsController dosite = Plausible.Repo.preload(conn.assigns.site, :owner)with :ok <- Plausible.Billing.Feature.Funnels.check_availability(site.owner),- query <- Query.from(site, params),+ query <- Query.from(site, params, debug_metadata(conn)),:ok <- validate_funnel_query(query),{funnel_id, ""} <- Integer.parse(funnel_id),{:ok, funnel} <- Stats.funnel(site, query, funnel_id) do@@ -586,7 +586,7 @@ defmodule PlausibleWeb.Api.StatsController dodef utm_mediums(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "property", "visit:utm_medium")- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))pagination = parse_pagination(params)metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])@@ -613,7 +613,7 @@ defmodule PlausibleWeb.Api.StatsController dodef utm_campaigns(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "property", "visit:utm_campaign")- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))pagination = parse_pagination(params)metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])@@ -640,7 +640,7 @@ defmodule PlausibleWeb.Api.StatsController dodef utm_contents(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "property", "visit:utm_content")- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))pagination = parse_pagination(params)metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])@@ -667,7 +667,7 @@ defmodule PlausibleWeb.Api.StatsController dodef utm_terms(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "property", "visit:utm_term")- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))pagination = parse_pagination(params)metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])@@ -694,7 +694,7 @@ defmodule PlausibleWeb.Api.StatsController dodef utm_sources(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "property", "visit:utm_source")- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))pagination = parse_pagination(params)metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])@@ -721,7 +721,7 @@ defmodule PlausibleWeb.Api.StatsController dodef referrers(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "property", "visit:referrer")- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))pagination = parse_pagination(params)metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])@@ -748,7 +748,7 @@ defmodule PlausibleWeb.Api.StatsController dodef referrer_drilldown(conn, %{"referrer" => "Google"} = params) dosite = conn.assigns[:site]- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))user_id = get_session(conn, :current_user_id)is_admin = user_id && Plausible.Sites.has_admin_access?(user_id, site)@@ -803,7 +803,7 @@ defmodule PlausibleWeb.Api.StatsController doreferrer_filter = LegacyDashboardFilterParser.filter_value("visit:source", referrer)query =- Query.from(site, params)+ Query.from(site, params, debug_metadata(conn))|> Query.add_filter(referrer_filter)pagination = parse_pagination(params)@@ -826,7 +826,7 @@ defmodule PlausibleWeb.Api.StatsController dodef pages(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "property", "event:page")- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))extra_metrics =if params["detailed"],@@ -859,7 +859,7 @@ defmodule PlausibleWeb.Api.StatsController dodef entry_pages(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "property", "visit:entry_page")- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))pagination = parse_pagination(params)metrics = breakdown_metrics(query, [:visits, :visit_duration])@@ -893,7 +893,7 @@ defmodule PlausibleWeb.Api.StatsController dodef exit_pages(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "property", "visit:exit_page")- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn)){limit, page} = parse_pagination(params)metrics = breakdown_metrics(query, [:visits])@@ -958,7 +958,7 @@ defmodule PlausibleWeb.Api.StatsController dodef countries(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "property", "visit:country")- query = site |> Query.from(params)+ query = Query.from(site, params, debug_metadata(conn))pagination = parse_pagination(params)metrics = breakdown_metrics(query, [:percentage])@@ -1013,7 +1013,7 @@ defmodule PlausibleWeb.Api.StatsController dodef regions(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "property", "visit:region")- query = site |> Query.from(params)+ query = Query.from(site, params, debug_metadata(conn))pagination = parse_pagination(params)metrics = breakdown_metrics(query)@@ -1051,7 +1051,7 @@ defmodule PlausibleWeb.Api.StatsController dodef cities(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "property", "visit:city")- query = site |> Query.from(params)+ query = Query.from(site, params, debug_metadata(conn))pagination = parse_pagination(params)metrics = breakdown_metrics(query)@@ -1094,7 +1094,7 @@ defmodule PlausibleWeb.Api.StatsController dodef browsers(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "property", "visit:browser")- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))pagination = parse_pagination(params)extra_metrics =@@ -1125,7 +1125,7 @@ defmodule PlausibleWeb.Api.StatsController dodef browser_versions(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "property", "visit:browser_version")- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))pagination = parse_pagination(params)extra_metrics =@@ -1165,7 +1165,7 @@ defmodule PlausibleWeb.Api.StatsController dodef operating_systems(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "property", "visit:os")- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))pagination = parse_pagination(params)extra_metrics =@@ -1196,7 +1196,7 @@ defmodule PlausibleWeb.Api.StatsController dodef operating_system_versions(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "property", "visit:os_version")- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))pagination = parse_pagination(params)extra_metrics =@@ -1236,7 +1236,7 @@ defmodule PlausibleWeb.Api.StatsController dodef screen_sizes(conn, params) dosite = conn.assigns[:site]params = Map.put(params, "property", "visit:device")- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))pagination = parse_pagination(params)extra_metrics =@@ -1268,7 +1268,7 @@ defmodule PlausibleWeb.Api.StatsController dopagination = parse_pagination(params)site = Plausible.Repo.preload(conn.assigns.site, :goals)params = Map.put(params, "property", "event:goal")- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))query =if query.period == "realtime" do@@ -1309,7 +1309,7 @@ defmodule PlausibleWeb.Api.StatsController docase Plausible.Props.ensure_prop_key_accessible(prop_key, site.owner) do:ok ->- json(conn, breakdown_custom_prop_values(site, params))+ json(conn, breakdown_custom_prop_values(conn, site, params)){:error, :upgrade_required} ->H.payment_required(@@ -1321,7 +1321,7 @@ defmodule PlausibleWeb.Api.StatsController dodef all_custom_prop_values(conn, params) dosite = conn.assigns.site- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))prop_names = Plausible.Stats.CustomProps.fetch_prop_names(site, query)@@ -1336,7 +1336,7 @@ defmodule PlausibleWeb.Api.StatsController dovalues =prop_names|> Enum.map(fn prop_key ->- breakdown_custom_prop_values(site, Map.put(params, "prop_key", prop_key))+ breakdown_custom_prop_values(conn, site, Map.put(params, "prop_key", prop_key))|> Map.get(:results)|> Enum.map(&Map.put(&1, :property, prop_key))|> transform_keys(%{:name => :value})@@ -1352,13 +1352,13 @@ defmodule PlausibleWeb.Api.StatsController doendend- defp breakdown_custom_prop_values(site, %{"prop_key" => prop_key} = params) do+ defp breakdown_custom_prop_values(conn, site, %{"prop_key" => prop_key} = params) dopagination = parse_pagination(params)prefixed_prop = "event:props:" <> prop_keyparams = Map.put(params, "property", prefixed_prop)- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))metrics =if Query.get_filter(query, "event:goal") do@@ -1388,7 +1388,7 @@ defmodule PlausibleWeb.Api.StatsController dodef filter_suggestions(conn, params) dosite = conn.assigns[:site]- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))json(conn,diff --git a/lib/plausible_web/controllers/debug_controller.ex b/lib/plausible_web/controllers/debug_controller.exnew file mode 100644index 0000000000..fb6e6a5524--- /dev/null+++ b/lib/plausible_web/controllers/debug_controller.ex@@ -0,0 +1,74 @@+defmodule PlausibleWeb.DebugController do+ use PlausibleWeb, :controller+ use Plausible.IngestRepo+ use Plausible++ import Ecto.Query++ plug(PlausibleWeb.RequireAccountPlug)+ plug(PlausibleWeb.SuperAdminOnlyPlug)++ def clickhouse(conn, params) do+ cluster? = Plausible.IngestRepo.clustered_table?("events_v2")+ on_cluster = if(cluster?, do: "ON CLUSTER '{cluster}'", else: "")++ # Ensure last logs are flushed+ IngestRepo.query("SYSTEM FLUSH LOGS #{on_cluster}")++ queries =+ from(+ l in from_query_log(cluster?),+ select: %{+ query: fragment("formatQuery(?)", l.query),+ log_comment: l.log_comment,+ type: l.type,+ event_time: l.event_time,+ query_duration_ms: l.query_duration_ms,+ query_id: l.query_id,+ memory_usage: fragment("formatReadableSize(?)", l.memory_usage),+ read_bytes: fragment("formatReadableSize(?)", l.read_bytes),+ result_bytes: fragment("formatReadableSize(?)", l.result_bytes),+ result_rows: l.result_rows+ },+ where: l.type > 1 and fragment("event_time > now() - toIntervalMinute(15)"),+ order_by: [desc: l.event_time]+ )+ |> filter_by_params(conn, params)+ |> IngestRepo.all()+ |> Enum.map(fn data ->+ Jason.decode!(data.log_comment)+ |> Map.merge(data)+ |> Map.delete(:log_comment)+ end)++ conn+ |> render("clickhouse.html",+ queries: queries,+ layout: {PlausibleWeb.LayoutView, "focus.html"}+ )+ end++ defp from_query_log(cluster?) do+ case cluster? do+ true -> from(l in fragment("clusterAllReplicas('{cluster}', system.query_log)"))+ false -> from(l in fragment("system.query_log"))+ end+ end++ defp filter_by_params(q, _conn, %{"user_id" => user_id}),+ do: where(q, [l], fragment("JSONExtractInt(?, \'user_id\') = ?", l.log_comment, ^user_id))++ defp filter_by_params(q, _conn, %{"site_id" => site_id}),+ do: where(q, [l], fragment("JSONExtractInt(?, \'site_id\') = ?", l.log_comment, ^site_id))++ defp filter_by_params(q, _conn, %{"site_domain" => site_domain}),+ do:+ where(+ q,+ [l],+ fragment("JSONExtractInt(?, \'site_domain\') = ?", l.log_comment, ^site_domain)+ )++ defp filter_by_params(q, conn, _),+ do: filter_by_params(q, conn, %{"user_id" => conn.assigns.current_user.id})+enddiff --git a/lib/plausible_web/controllers/helpers.ex b/lib/plausible_web/controllers/helpers.exindex e8a1d5976a..e6c3edea66 100644--- a/lib/plausible_web/controllers/helpers.ex+++ b/lib/plausible_web/controllers/helpers.ex@@ -18,4 +18,20 @@ defmodule PlausibleWeb.ControllerHelpers dodefp error_layout,do: Application.get_env(:plausible, PlausibleWeb.Endpoint)[:render_errors][:layout]++ def debug_metadata(conn) do+ %{+ request_method: conn.method,+ request_path: conn.request_path,+ params: conn.params,+ phoenix_controller: conn.private.phoenix_controller |> to_string(),+ phoenix_action: conn.private.phoenix_action |> to_string(),+ site_id: conn.assigns.site.id,+ site_domain: conn.assigns.site.domain,+ user_id: get_user_id(conn, conn.assigns)+ }+ end++ defp get_user_id(_conn, %{current_user: user}), do: user.id+ defp get_user_id(conn, _assigns), do: get_session(conn, :current_user_id)enddiff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.exindex b2bb5eedc1..104de278c9 100644--- a/lib/plausible_web/controllers/stats_controller.ex+++ b/lib/plausible_web/controllers/stats_controller.ex@@ -112,7 +112,7 @@ defmodule PlausibleWeb.StatsController dodef csv_export(conn, params) doif is_nil(params["interval"]) or Plausible.Stats.Interval.valid?(params["interval"]) dosite = Plausible.Repo.preload(conn.assigns.site, :owner)- query = Query.from(site, params)+ query = Query.from(site, params, debug_metadata(conn))filename =~c"Plausible export #{params["domain"]} #{Date.to_iso8601(query.date_range.first)} to #{Date.to_iso8601(query.date_range.last)} .zip"diff --git a/lib/plausible_web/plugs/crm_auth_plug.ex b/lib/plausible_web/plugs/super_admin_only_plug.exsimilarity index 88%rename from lib/plausible_web/plugs/crm_auth_plug.exrename to lib/plausible_web/plugs/super_admin_only_plug.exindex fa68864d8e..812ed1d2c5 100644--- a/lib/plausible_web/plugs/crm_auth_plug.ex+++ b/lib/plausible_web/plugs/super_admin_only_plug.ex@@ -1,4 +1,6 @@-defmodule PlausibleWeb.CRMAuthPlug do+defmodule PlausibleWeb.SuperAdminOnlyPlug do+ @moduledoc false+import Plug.Connuse Plausible.Repodiff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.exindex 16db14e346..3857c44786 100644--- a/lib/plausible_web/router.ex+++ b/lib/plausible_web/router.ex@@ -58,7 +58,7 @@ defmodule PlausibleWeb.Router doplug PlausibleWeb.Plugs.NoRobotsplug :fetch_session- plug PlausibleWeb.CRMAuthPlug+ plug PlausibleWeb.SuperAdminOnlyPlugendend@@ -69,7 +69,7 @@ defmodule PlausibleWeb.Router doon_ee douse Kaffy.Routes,scope: "/crm",- pipe_through: [PlausibleWeb.Plugs.NoRobots, PlausibleWeb.CRMAuthPlug]+ pipe_through: [PlausibleWeb.Plugs.NoRobots, PlausibleWeb.SuperAdminOnlyPlug]endon_ee do@@ -429,6 +429,8 @@ defmodule PlausibleWeb.Router doget "/:website/download/export", SiteController, :download_exportget "/:website/settings/import", SiteController, :csv_import+ get "/debug/clickhouse", DebugController, :clickhouse+get "/:domain/export", StatsController, :csv_exportget "/:domain/*path", StatsController, :statsenddiff --git a/lib/plausible_web/templates/debug/clickhouse.html.heex b/lib/plausible_web/templates/debug/clickhouse.html.heexnew file mode 100644index 0000000000..cfe9304fd1--- /dev/null+++ b/lib/plausible_web/templates/debug/clickhouse.html.heex@@ -0,0 +1,49 @@+<div class="w-full max-w-4xl bg-white dark:bg-gray-800 shadow-md rounded p-8 mb-4 mt-8 mx-auto">+ <section class="grid grid-cols-1 gap-y-3 divide-y">+ <%= for log <- @queries do %>+ <details class="group py-1">+ <summary class="flex cursor-pointer flex-row items-center justify-between py-1 font-semibold text-gray-600 marker:[font-size:0px]">+ <%= log["request_method"] %> <%= controller_name(log["phoenix_controller"]) %>.<%= log[+ "phoenix_action"+ ] %> (<%= log[:query_duration_ms] %>ms)+ <svg+ class="h-6 w-6 rotate-0 transform text-gray-400 group-open:rotate-180"+ xmlns="http://www.w3.org/2000/svg"+ fill="none"+ viewBox="0 0 24 24"+ stroke-width="2"+ stroke="currentColor"+ aria-hidden="true"+ >+ <path stroke-linecap="round" stroke-linejoin="round" d="M19 9l-7 7-7-7"></path>+ </svg>+ </summary>+ <table class="table table-striped table-auto">+ <tbody>+ <%= for {key, value} <- log do %>+ <tr class="table-row">+ <td class="table-cell p-2"><%= key %></td>+ <td class="table-cell p-2">+ <%= case key do %>+ <% :query -> %>+ <pre class="whitespace-pre-wrap"><%= value %></pre>+ <% "params" -> %>+ <pre><%= Jason.encode!(value, pretty: true) %></pre>+ <% _ -> %>+ <%= value %>+ <% end %>+ </td>+ </tr>+ <% end %>+ </tbody>+ </table>+ </details>+ <% end %>+ </section>++ <%= if length(@queries) == 0 do %>+ <p class="text-gray-500">+ No queries run by user in the last few minutes. Try opening a dashboard+ </p>+ <% end %>+</div>diff --git a/lib/plausible_web/views/debug_view.ex b/lib/plausible_web/views/debug_view.exnew file mode 100644index 0000000000..4e8ce74077--- /dev/null+++ b/lib/plausible_web/views/debug_view.ex@@ -0,0 +1,11 @@+defmodule PlausibleWeb.DebugView do+ use PlausibleWeb, :view++ def controller_name(phoenix_controller_name) do+ phoenix_controller_name+ |> String.to_existing_atom()+ |> Module.split()+ |> Enum.drop_while(&String.starts_with?(&1, "Plausible"))+ |> Enum.join(".")+ end+enddiff --git a/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs b/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exsindex 3c535d3cae..0cc2f76b7f 100644--- a/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs+++ b/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs@@ -6,7 +6,7 @@ defmodule Plausible.IngestRepo.Migrations.AddImportedCustomEvents doon_cluster = Plausible.MigrationUtils.on_cluster_statement("imported_pages")settings =- if Plausible.MigrationUtils.clustered_table?("imported_pages") do+ if Plausible.IngestRepo.clustered_table?("imported_pages") do"""ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/{database}/imported_custom_events', '{replica}')ORDER BY (site_id, import_id, date, name)