Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Populate log_comment with debug information, /debug/clickhouse route (#4435)

Karl-Aksel Puulmann496796027822
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bc1bb3c11f..83cbe40a4f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,8 @@ All notable changes to this project will be documented in this file.
- Add `does_not_contain` filter support to dashboard
- Traffic drop notifications plausible/analytics#4300
- Add search and pagination functionality into Google Keywords > Details modal
+- ClickHouse system.query_log table log_comment column now contains information about source of queries. Useful for debugging
+- New /debug/clickhouse route for super admins which shows information on clickhouse queries executed by user
### Removed
- Deprecate `ECTO_IPV6` and `ECTO_CH_IPV6` env vars in CE plausible/analytics#4245
diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.ex
index 37d5961cb9..35821d8d46 100644
--- a/lib/plausible/clickhouse_repo.ex
+++ b/lib/plausible/clickhouse_repo.ex
@@ -30,4 +30,17 @@ defmodule Plausible.ClickhouseRepo do
|> Enum.to_list()
|> Keyword.values()
end
+
+ @impl true
+ def prepare_query(_operation, query, opts) do
+ {plausible_query, opts} = Keyword.pop(opts, :query)
+ log_comment = if(plausible_query, do: Jason.encode!(plausible_query.debug_metadata), else: "")
+
+ opts =
+ Keyword.update(opts, :settings, [log_comment: log_comment], fn settings ->
+ [{:log_comment, log_comment} | settings]
+ end)
+
+ {query, opts}
+ end
end
diff --git a/lib/plausible/data_migration/locations_sync.ex b/lib/plausible/data_migration/locations_sync.ex
index 9b81179c84..7ebba6098b 100644
--- a/lib/plausible/data_migration/locations_sync.ex
+++ b/lib/plausible/data_migration/locations_sync.ex
@@ -88,7 +88,7 @@ defmodule Plausible.DataMigration.LocationsSync do
end
def run() do
- cluster? = Plausible.MigrationUtils.clustered_table?("sessions_v2")
+ cluster? = Plausible.IngestRepo.clustered_table?("sessions_v2")
{:ok, _} = run_sql("truncate-location-data-table", cluster?: cluster?)
{:ok, _} = run_sql("create-location-data-table", cluster?: cluster?)
diff --git a/lib/plausible/data_migration/populate_event_session_columns.ex b/lib/plausible/data_migration/populate_event_session_columns.ex
index 796fc0435d..9ae543a993 100644
--- a/lib/plausible/data_migration/populate_event_session_columns.ex
+++ b/lib/plausible/data_migration/populate_event_session_columns.ex
@@ -22,7 +22,7 @@ defmodule Plausible.DataMigration.PopulateEventSessionColumns do
}
def run(opts \\ []) do
- cluster? = Plausible.MigrationUtils.clustered_table?("sessions_v2")
+ cluster? = Plausible.IngestRepo.clustered_table?("sessions_v2")
{:ok, _} =
run_sql("create-sessions-dictionary",
@@ -54,7 +54,7 @@ defmodule Plausible.DataMigration.PopulateEventSessionColumns do
end
def kill(opts \\ []) do
- cluster? = Plausible.MigrationUtils.clustered_table?("events_v2")
+ cluster? = Plausible.IngestRepo.clustered_table?("events_v2")
report_progress(opts)
diff --git a/lib/plausible/data_migration/versioned_sessions.ex b/lib/plausible/data_migration/versioned_sessions.ex
index 7152d2d23b..4288683c9d 100644
--- a/lib/plausible/data_migration/versioned_sessions.ex
+++ b/lib/plausible/data_migration/versioned_sessions.ex
@@ -21,7 +21,7 @@ defmodule Plausible.DataMigration.VersionedSessions do
unique_suffix = Timex.now() |> Calendar.strftime(@suffix_format)
- cluster? = Plausible.MigrationUtils.clustered_table?("sessions_v2")
+ cluster? = Plausible.IngestRepo.clustered_table?("sessions_v2")
{:ok, %{rows: partitions}} = run_sql("list-partitions")
partitions = Enum.map(partitions, fn [part] -> part end)
diff --git a/lib/plausible/ingest_repo.ex b/lib/plausible/ingest_repo.ex
index 3fe099f10f..703082e1ce 100644
--- a/lib/plausible/ingest_repo.ex
+++ b/lib/plausible/ingest_repo.ex
@@ -14,4 +14,11 @@ defmodule Plausible.IngestRepo do
import Ecto.Query, only: [from: 1, from: 2]
end
end
+
+ def clustered_table?(table) do
+ case query("SELECT 1 FROM system.replicas WHERE table = '#{table}'") do
+ {:ok, %{rows: []}} -> false
+ {:ok, _} -> true
+ end
+ end
end
diff --git a/lib/plausible/migration_utils.ex b/lib/plausible/migration_utils.ex
index a5f4eba6bf..e477537e76 100644
--- a/lib/plausible/migration_utils.ex
+++ b/lib/plausible/migration_utils.ex
@@ -3,20 +3,15 @@ defmodule Plausible.MigrationUtils do
Base module for to use in Clickhouse migrations
"""
- def on_cluster_statement(table) do
- if(clustered_table?(table), do: "ON CLUSTER '{cluster}'", else: "")
- end
+ alias Plausible.IngestRepo
- def clustered_table?(table) do
- case Plausible.IngestRepo.query("SELECT 1 FROM system.replicas WHERE table = '#{table}'") do
- {:ok, %{rows: []}} -> false
- {:ok, _} -> true
- end
+ def on_cluster_statement(table) do
+ if(IngestRepo.clustered_table?(table), do: "ON CLUSTER '{cluster}'", else: "")
end
# See https://clickhouse.com/docs/en/sql-reference/dictionaries#clickhouse for context
def dictionary_connection_params() do
- Plausible.IngestRepo.config()
+ IngestRepo.config()
|> Enum.map(fn
{:database, database} -> "DB '#{database}'"
{:username, username} -> "USER '#{username}'"
diff --git a/lib/plausible/stats.ex b/lib/plausible/stats.ex
index 2db853b4dd..0baa6b9c07 100644
--- a/lib/plausible/stats.ex
+++ b/lib/plausible/stats.ex
@@ -22,7 +22,7 @@ defmodule Plausible.Stats do
optimized_query
|> SQL.QueryBuilder.build(site)
- |> ClickhouseRepo.all()
+ |> ClickhouseRepo.all(query: query)
|> QueryResult.from(optimized_query)
end
diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex
index 790b5ed719..bcf1faffa8 100644
--- a/lib/plausible/stats/aggregate.ex
+++ b/lib/plausible/stats/aggregate.ex
@@ -33,7 +33,7 @@ defmodule Plausible.Stats.Aggregate do
end
Plausible.ClickhouseRepo.parallel_tasks([
- run_query_task(q),
+ run_query_task(q, query),
time_on_page_task
])
|> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end)
@@ -44,8 +44,8 @@ defmodule Plausible.Stats.Aggregate do
|> Enum.into(%{})
end
- defp run_query_task(nil), do: fn -> %{} end
- defp run_query_task(q), do: fn -> ClickhouseRepo.one(q) end
+ defp run_query_task(nil, _query), do: fn -> %{} end
+ defp run_query_task(q, query), do: fn -> ClickhouseRepo.one(q, query: query) end
defp aggregate_time_on_page(site, query) do
windowed_pages_q =
@@ -87,7 +87,7 @@ defmodule Plausible.Stats.Aggregate do
from e in Ecto.Query.subquery(avg_time_per_page_transition_q),
select: fragment("avg(ifNotFinite(?,NULL))", e.avg)
- %{time_on_page: ClickhouseRepo.one(time_on_page_q)}
+ %{time_on_page: ClickhouseRepo.one(time_on_page_q, query: query)}
end
@metrics_to_round [:bounce_rate, :time_on_page, :visit_duration, :sample_percent]
diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index 05c150a076..fa6c576185 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -33,7 +33,7 @@ defmodule Plausible.Stats.Breakdown do
q
|> apply_pagination(pagination)
- |> ClickhouseRepo.all()
+ |> ClickhouseRepo.all(query: query)
|> QueryResult.from(query_with_metrics)
|> build_breakdown_result(query_with_metrics, metrics)
|> maybe_add_time_on_page(site, query_with_metrics, metrics)
@@ -150,7 +150,7 @@ defmodule Plausible.Stats.Breakdown do
end
timed_pages_q
- |> Plausible.ClickhouseRepo.all()
+ |> Plausible.ClickhouseRepo.all(query: query)
|> Map.new()
end
diff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.ex
index 02d02c9f29..3abd8194c5 100644
--- a/lib/plausible/stats/legacy/legacy_query_builder.ex
+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex
@@ -5,12 +5,12 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
alias Plausible.Stats.{Filters, Interval, Query}
- def from(site, params) do
+ def from(site, params, debug_metadata) do
now = NaiveDateTime.utc_now(:second)
query =
Query
- |> struct!(now: now, timezone: site.timezone)
+ |> struct!(now: now, timezone: site.timezone, debug_metadata: debug_metadata)
|> put_period(site, params)
|> put_dimensions(params)
|> put_interval(params)
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index 7de4e931b4..5bfcbf7c99 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -21,20 +21,21 @@ defmodule Plausible.Stats.Query do
include: %{
imports: false,
time_labels: false
- }
+ },
+ debug_metadata: %{}
require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{Filters, Imported, Legacy}
@type t :: %__MODULE__{}
- def build(site, params) do
+ def build(site, params, debug_metadata) do
with {:ok, query_data} <- Filters.QueryParser.parse(site, params) do
query =
struct!(__MODULE__, Map.to_list(query_data))
|> put_imported_opts(site, %{})
|> put_experimental_reduced_joins(site, params)
- |> struct!(v2: true, now: NaiveDateTime.utc_now(:second))
+ |> struct!(v2: true, now: NaiveDateTime.utc_now(:second), debug_metadata: debug_metadata)
{:ok, query}
end
@@ -43,9 +44,8 @@ defmodule Plausible.Stats.Query do
@doc """
Builds query from old-style params. New code should prefer Query.build
"""
- @spec from(Plausible.Site.t(), map()) :: t()
- def from(site, params) do
- Legacy.QueryBuilder.from(site, params)
+ def from(site, params, debug_metadata \\ %{}) do
+ Legacy.QueryBuilder.from(site, params, debug_metadata)
end
def put_experimental_reduced_joins(query, site, params) do
diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex
index 513debc597..6971fe7dcf 100644
--- a/lib/plausible/stats/timeseries.ex
+++ b/lib/plausible/stats/timeseries.ex
@@ -39,7 +39,7 @@ defmodule Plausible.Stats.Timeseries do
q = SQL.QueryBuilder.build(query_with_metrics, site)
q
- |> ClickhouseRepo.all()
+ |> ClickhouseRepo.all(query: query)
|> QueryResult.from(query_with_metrics)
|> build_timeseries_result(query_with_metrics, currency)
|> transform_keys(%{group_conversion_rate: :conversion_rate})
diff --git a/lib/plausible_web/controllers/api/external_query_api_controller.ex b/lib/plausible_web/controllers/api/external_query_api_controller.ex
index d2bf01401f..5b88d39e7c 100644
--- a/lib/plausible_web/controllers/api/external_query_api_controller.ex
+++ b/lib/plausible_web/controllers/api/external_query_api_controller.ex
@@ -9,7 +9,7 @@ defmodule PlausibleWeb.Api.ExternalQueryApiController do
def query(conn, params) do
site = Repo.preload(conn.assigns.site, :owner)
- case Query.build(site, params) do
+ case Query.build(site, params, debug_metadata(conn)) do
{:ok, query} ->
results = Plausible.Stats.query(site, query)
json(conn, results)
diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex
index 1ba1a8079d..fa3188550d 100644
--- a/lib/plausible_web/controllers/api/external_stats_controller.ex
+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex
@@ -16,7 +16,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
with :ok <- validate_period(params),
:ok <- validate_date(params),
- query <- Query.from(site, params),
+ query <- Query.from(site, params, debug_metadata(conn)),
:ok <- validate_filters(site, query.filters),
{:ok, metrics} <- parse_and_validate_metrics(params, query),
:ok <- ensure_custom_props_access(site, query) do
@@ -55,7 +55,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
with :ok <- validate_period(params),
:ok <- validate_date(params),
:ok <- validate_property(params),
- query <- Query.from(site, params),
+ query <- Query.from(site, params, debug_metadata(conn)),
:ok <- validate_filters(site, query.filters),
{:ok, metrics} <- parse_and_validate_metrics(params, query),
{:ok, limit} <- validate_or_default_limit(params),
@@ -262,7 +262,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
with :ok <- validate_period(params),
:ok <- validate_date(params),
:ok <- validate_interval(params),
- query <- Query.from(site, params),
+ query <- Query.from(site, params, debug_metadata(conn)),
:ok <- validate_filters(site, query.filters),
{:ok, metrics} <- parse_and_validate_metrics(params, query),
:ok <- ensure_custom_props_access(site, query) do
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index 2114a973d1..8204abda2d 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -104,7 +104,7 @@ defmodule PlausibleWeb.Api.StatsController do
with {:ok, dates} <- parse_date_params(params),
:ok <- validate_interval(params),
:ok <- validate_interval_granularity(site, params, dates),
- query = Query.from(site, params),
+ query = Query.from(site, params, debug_metadata(conn)),
{:ok, metric} <- parse_and_validate_graph_metric(params, query) do
timeseries_query =
if query.period == "realtime" do
@@ -206,7 +206,7 @@ defmodule PlausibleWeb.Api.StatsController do
def top_stats(conn, params) do
site = conn.assigns[:site]
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
comparison_opts = parse_comparison_opts(params)
@@ -501,7 +501,7 @@ defmodule PlausibleWeb.Api.StatsController do
def sources(conn, params) do
site = conn.assigns[:site]
params = Map.put(params, "property", "visit:source")
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
pagination = parse_pagination(params)
extra_metrics =
@@ -534,7 +534,7 @@ defmodule PlausibleWeb.Api.StatsController do
site = Plausible.Repo.preload(conn.assigns.site, :owner)
with :ok <- Plausible.Billing.Feature.Funnels.check_availability(site.owner),
- query <- Query.from(site, params),
+ query <- Query.from(site, params, debug_metadata(conn)),
:ok <- validate_funnel_query(query),
{funnel_id, ""} <- Integer.parse(funnel_id),
{:ok, funnel} <- Stats.funnel(site, query, funnel_id) do
@@ -586,7 +586,7 @@ defmodule PlausibleWeb.Api.StatsController do
def utm_mediums(conn, params) do
site = conn.assigns[:site]
params = Map.put(params, "property", "visit:utm_medium")
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
pagination = parse_pagination(params)
metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])
@@ -613,7 +613,7 @@ defmodule PlausibleWeb.Api.StatsController do
def utm_campaigns(conn, params) do
site = conn.assigns[:site]
params = Map.put(params, "property", "visit:utm_campaign")
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
pagination = parse_pagination(params)
metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])
@@ -640,7 +640,7 @@ defmodule PlausibleWeb.Api.StatsController do
def utm_contents(conn, params) do
site = conn.assigns[:site]
params = Map.put(params, "property", "visit:utm_content")
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
pagination = parse_pagination(params)
metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])
@@ -667,7 +667,7 @@ defmodule PlausibleWeb.Api.StatsController do
def utm_terms(conn, params) do
site = conn.assigns[:site]
params = Map.put(params, "property", "visit:utm_term")
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
pagination = parse_pagination(params)
metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])
@@ -694,7 +694,7 @@ defmodule PlausibleWeb.Api.StatsController do
def utm_sources(conn, params) do
site = conn.assigns[:site]
params = Map.put(params, "property", "visit:utm_source")
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
pagination = parse_pagination(params)
metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])
@@ -721,7 +721,7 @@ defmodule PlausibleWeb.Api.StatsController do
def referrers(conn, params) do
site = conn.assigns[:site]
params = Map.put(params, "property", "visit:referrer")
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
pagination = parse_pagination(params)
metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])
@@ -748,7 +748,7 @@ defmodule PlausibleWeb.Api.StatsController do
def referrer_drilldown(conn, %{"referrer" => "Google"} = params) do
site = conn.assigns[:site]
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
user_id = get_session(conn, :current_user_id)
is_admin = user_id && Plausible.Sites.has_admin_access?(user_id, site)
@@ -803,7 +803,7 @@ defmodule PlausibleWeb.Api.StatsController do
referrer_filter = LegacyDashboardFilterParser.filter_value("visit:source", referrer)
query =
- Query.from(site, params)
+ Query.from(site, params, debug_metadata(conn))
|> Query.add_filter(referrer_filter)
pagination = parse_pagination(params)
@@ -826,7 +826,7 @@ defmodule PlausibleWeb.Api.StatsController do
def pages(conn, params) do
site = conn.assigns[:site]
params = Map.put(params, "property", "event:page")
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
extra_metrics =
if params["detailed"],
@@ -859,7 +859,7 @@ defmodule PlausibleWeb.Api.StatsController do
def entry_pages(conn, params) do
site = conn.assigns[:site]
params = Map.put(params, "property", "visit:entry_page")
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
pagination = parse_pagination(params)
metrics = breakdown_metrics(query, [:visits, :visit_duration])
@@ -893,7 +893,7 @@ defmodule PlausibleWeb.Api.StatsController do
def exit_pages(conn, params) do
site = conn.assigns[:site]
params = Map.put(params, "property", "visit:exit_page")
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
{limit, page} = parse_pagination(params)
metrics = breakdown_metrics(query, [:visits])
@@ -958,7 +958,7 @@ defmodule PlausibleWeb.Api.StatsController do
def countries(conn, params) do
site = conn.assigns[:site]
params = Map.put(params, "property", "visit:country")
- query = site |> Query.from(params)
+ query = Query.from(site, params, debug_metadata(conn))
pagination = parse_pagination(params)
metrics = breakdown_metrics(query, [:percentage])
@@ -1013,7 +1013,7 @@ defmodule PlausibleWeb.Api.StatsController do
def regions(conn, params) do
site = conn.assigns[:site]
params = Map.put(params, "property", "visit:region")
- query = site |> Query.from(params)
+ query = Query.from(site, params, debug_metadata(conn))
pagination = parse_pagination(params)
metrics = breakdown_metrics(query)
@@ -1051,7 +1051,7 @@ defmodule PlausibleWeb.Api.StatsController do
def cities(conn, params) do
site = conn.assigns[:site]
params = Map.put(params, "property", "visit:city")
- query = site |> Query.from(params)
+ query = Query.from(site, params, debug_metadata(conn))
pagination = parse_pagination(params)
metrics = breakdown_metrics(query)
@@ -1094,7 +1094,7 @@ defmodule PlausibleWeb.Api.StatsController do
def browsers(conn, params) do
site = conn.assigns[:site]
params = Map.put(params, "property", "visit:browser")
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
pagination = parse_pagination(params)
extra_metrics =
@@ -1125,7 +1125,7 @@ defmodule PlausibleWeb.Api.StatsController do
def browser_versions(conn, params) do
site = conn.assigns[:site]
params = Map.put(params, "property", "visit:browser_version")
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
pagination = parse_pagination(params)
extra_metrics =
@@ -1165,7 +1165,7 @@ defmodule PlausibleWeb.Api.StatsController do
def operating_systems(conn, params) do
site = conn.assigns[:site]
params = Map.put(params, "property", "visit:os")
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
pagination = parse_pagination(params)
extra_metrics =
@@ -1196,7 +1196,7 @@ defmodule PlausibleWeb.Api.StatsController do
def operating_system_versions(conn, params) do
site = conn.assigns[:site]
params = Map.put(params, "property", "visit:os_version")
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
pagination = parse_pagination(params)
extra_metrics =
@@ -1236,7 +1236,7 @@ defmodule PlausibleWeb.Api.StatsController do
def screen_sizes(conn, params) do
site = conn.assigns[:site]
params = Map.put(params, "property", "visit:device")
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
pagination = parse_pagination(params)
extra_metrics =
@@ -1268,7 +1268,7 @@ defmodule PlausibleWeb.Api.StatsController do
pagination = parse_pagination(params)
site = Plausible.Repo.preload(conn.assigns.site, :goals)
params = Map.put(params, "property", "event:goal")
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
query =
if query.period == "realtime" do
@@ -1309,7 +1309,7 @@ defmodule PlausibleWeb.Api.StatsController do
case Plausible.Props.ensure_prop_key_accessible(prop_key, site.owner) do
:ok ->
- json(conn, breakdown_custom_prop_values(site, params))
+ json(conn, breakdown_custom_prop_values(conn, site, params))
{:error, :upgrade_required} ->
H.payment_required(
@@ -1321,7 +1321,7 @@ defmodule PlausibleWeb.Api.StatsController do
def all_custom_prop_values(conn, params) do
site = conn.assigns.site
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
prop_names = Plausible.Stats.CustomProps.fetch_prop_names(site, query)
@@ -1336,7 +1336,7 @@ defmodule PlausibleWeb.Api.StatsController do
values =
prop_names
|> Enum.map(fn prop_key ->
- breakdown_custom_prop_values(site, Map.put(params, "prop_key", prop_key))
+ breakdown_custom_prop_values(conn, site, Map.put(params, "prop_key", prop_key))
|> Map.get(:results)
|> Enum.map(&Map.put(&1, :property, prop_key))
|> transform_keys(%{:name => :value})
@@ -1352,13 +1352,13 @@ defmodule PlausibleWeb.Api.StatsController do
end
end
- defp breakdown_custom_prop_values(site, %{"prop_key" => prop_key} = params) do
+ defp breakdown_custom_prop_values(conn, site, %{"prop_key" => prop_key} = params) do
pagination = parse_pagination(params)
prefixed_prop = "event:props:" <> prop_key
params = Map.put(params, "property", prefixed_prop)
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
metrics =
if Query.get_filter(query, "event:goal") do
@@ -1388,7 +1388,7 @@ defmodule PlausibleWeb.Api.StatsController do
def filter_suggestions(conn, params) do
site = conn.assigns[:site]
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
json(
conn,
diff --git a/lib/plausible_web/controllers/debug_controller.ex b/lib/plausible_web/controllers/debug_controller.ex
new file mode 100644
index 0000000000..fb6e6a5524
--- /dev/null
+++ b/lib/plausible_web/controllers/debug_controller.ex
@@ -0,0 +1,74 @@
+defmodule PlausibleWeb.DebugController do
+ use PlausibleWeb, :controller
+ use Plausible.IngestRepo
+ use Plausible
+
+ import Ecto.Query
+
+ plug(PlausibleWeb.RequireAccountPlug)
+ plug(PlausibleWeb.SuperAdminOnlyPlug)
+
+ def clickhouse(conn, params) do
+ cluster? = Plausible.IngestRepo.clustered_table?("events_v2")
+ on_cluster = if(cluster?, do: "ON CLUSTER '{cluster}'", else: "")
+
+ # Ensure last logs are flushed
+ IngestRepo.query("SYSTEM FLUSH LOGS #{on_cluster}")
+
+ queries =
+ from(
+ l in from_query_log(cluster?),
+ select: %{
+ query: fragment("formatQuery(?)", l.query),
+ log_comment: l.log_comment,
+ type: l.type,
+ event_time: l.event_time,
+ query_duration_ms: l.query_duration_ms,
+ query_id: l.query_id,
+ memory_usage: fragment("formatReadableSize(?)", l.memory_usage),
+ read_bytes: fragment("formatReadableSize(?)", l.read_bytes),
+ result_bytes: fragment("formatReadableSize(?)", l.result_bytes),
+ result_rows: l.result_rows
+ },
+ where: l.type > 1 and fragment("event_time > now() - toIntervalMinute(15)"),
+ order_by: [desc: l.event_time]
+ )
+ |> filter_by_params(conn, params)
+ |> IngestRepo.all()
+ |> Enum.map(fn data ->
+ Jason.decode!(data.log_comment)
+ |> Map.merge(data)
+ |> Map.delete(:log_comment)
+ end)
+
+ conn
+ |> render("clickhouse.html",
+ queries: queries,
+ layout: {PlausibleWeb.LayoutView, "focus.html"}
+ )
+ end
+
+ defp from_query_log(cluster?) do
+ case cluster? do
+ true -> from(l in fragment("clusterAllReplicas('{cluster}', system.query_log)"))
+ false -> from(l in fragment("system.query_log"))
+ end
+ end
+
+ defp filter_by_params(q, _conn, %{"user_id" => user_id}),
+ do: where(q, [l], fragment("JSONExtractInt(?, \'user_id\') = ?", l.log_comment, ^user_id))
+
+ defp filter_by_params(q, _conn, %{"site_id" => site_id}),
+ do: where(q, [l], fragment("JSONExtractInt(?, \'site_id\') = ?", l.log_comment, ^site_id))
+
+ defp filter_by_params(q, _conn, %{"site_domain" => site_domain}),
+ do:
+ where(
+ q,
+ [l],
+ fragment("JSONExtractInt(?, \'site_domain\') = ?", l.log_comment, ^site_domain)
+ )
+
+ defp filter_by_params(q, conn, _),
+ do: filter_by_params(q, conn, %{"user_id" => conn.assigns.current_user.id})
+end
diff --git a/lib/plausible_web/controllers/helpers.ex b/lib/plausible_web/controllers/helpers.ex
index e8a1d5976a..e6c3edea66 100644
--- a/lib/plausible_web/controllers/helpers.ex
+++ b/lib/plausible_web/controllers/helpers.ex
@@ -18,4 +18,20 @@ defmodule PlausibleWeb.ControllerHelpers do
defp error_layout,
do: Application.get_env(:plausible, PlausibleWeb.Endpoint)[:render_errors][:layout]
+
+ def debug_metadata(conn) do
+ %{
+ request_method: conn.method,
+ request_path: conn.request_path,
+ params: conn.params,
+ phoenix_controller: conn.private.phoenix_controller |> to_string(),
+ phoenix_action: conn.private.phoenix_action |> to_string(),
+ site_id: conn.assigns.site.id,
+ site_domain: conn.assigns.site.domain,
+ user_id: get_user_id(conn, conn.assigns)
+ }
+ end
+
+ defp get_user_id(_conn, %{current_user: user}), do: user.id
+ defp get_user_id(conn, _assigns), do: get_session(conn, :current_user_id)
end
diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex
index b2bb5eedc1..104de278c9 100644
--- a/lib/plausible_web/controllers/stats_controller.ex
+++ b/lib/plausible_web/controllers/stats_controller.ex
@@ -112,7 +112,7 @@ defmodule PlausibleWeb.StatsController do
def csv_export(conn, params) do
if is_nil(params["interval"]) or Plausible.Stats.Interval.valid?(params["interval"]) do
site = Plausible.Repo.preload(conn.assigns.site, :owner)
- query = Query.from(site, params)
+ query = Query.from(site, params, debug_metadata(conn))
filename =
~c"Plausible export #{params["domain"]} #{Date.to_iso8601(query.date_range.first)} to #{Date.to_iso8601(query.date_range.last)} .zip"
diff --git a/lib/plausible_web/plugs/crm_auth_plug.ex b/lib/plausible_web/plugs/super_admin_only_plug.ex
similarity index 88%
rename from lib/plausible_web/plugs/crm_auth_plug.ex
rename to lib/plausible_web/plugs/super_admin_only_plug.ex
index fa68864d8e..812ed1d2c5 100644
--- a/lib/plausible_web/plugs/crm_auth_plug.ex
+++ b/lib/plausible_web/plugs/super_admin_only_plug.ex
@@ -1,4 +1,6 @@
-defmodule PlausibleWeb.CRMAuthPlug do
+defmodule PlausibleWeb.SuperAdminOnlyPlug do
+ @moduledoc false
+
import Plug.Conn
use Plausible.Repo
diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex
index 16db14e346..3857c44786 100644
--- a/lib/plausible_web/router.ex
+++ b/lib/plausible_web/router.ex
@@ -58,7 +58,7 @@ defmodule PlausibleWeb.Router do
plug PlausibleWeb.Plugs.NoRobots
plug :fetch_session
- plug PlausibleWeb.CRMAuthPlug
+ plug PlausibleWeb.SuperAdminOnlyPlug
end
end
@@ -69,7 +69,7 @@ defmodule PlausibleWeb.Router do
on_ee do
use Kaffy.Routes,
scope: "/crm",
- pipe_through: [PlausibleWeb.Plugs.NoRobots, PlausibleWeb.CRMAuthPlug]
+ pipe_through: [PlausibleWeb.Plugs.NoRobots, PlausibleWeb.SuperAdminOnlyPlug]
end
on_ee do
@@ -429,6 +429,8 @@ defmodule PlausibleWeb.Router do
get "/:website/download/export", SiteController, :download_export
get "/:website/settings/import", SiteController, :csv_import
+ get "/debug/clickhouse", DebugController, :clickhouse
+
get "/:domain/export", StatsController, :csv_export
get "/:domain/*path", StatsController, :stats
end
diff --git a/lib/plausible_web/templates/debug/clickhouse.html.heex b/lib/plausible_web/templates/debug/clickhouse.html.heex
new file mode 100644
index 0000000000..cfe9304fd1
--- /dev/null
+++ b/lib/plausible_web/templates/debug/clickhouse.html.heex
@@ -0,0 +1,49 @@
+<div class="w-full max-w-4xl bg-white dark:bg-gray-800 shadow-md rounded p-8 mb-4 mt-8 mx-auto">
+ <section class="grid grid-cols-1 gap-y-3 divide-y">
+ <%= for log <- @queries do %>
+ <details class="group py-1">
+ <summary class="flex cursor-pointer flex-row items-center justify-between py-1 font-semibold text-gray-600 marker:[font-size:0px]">
+ <%= log["request_method"] %> <%= controller_name(log["phoenix_controller"]) %>.<%= log[
+ "phoenix_action"
+ ] %> (<%= log[:query_duration_ms] %>ms)
+ <svg
+ class="h-6 w-6 rotate-0 transform text-gray-400 group-open:rotate-180"
+ xmlns="http://www.w3.org/2000/svg"
+ fill="none"
+ viewBox="0 0 24 24"
+ stroke-width="2"
+ stroke="currentColor"
+ aria-hidden="true"
+ >
+ <path stroke-linecap="round" stroke-linejoin="round" d="M19 9l-7 7-7-7"></path>
+ </svg>
+ </summary>
+ <table class="table table-striped table-auto">
+ <tbody>
+ <%= for {key, value} <- log do %>
+ <tr class="table-row">
+ <td class="table-cell p-2"><%= key %></td>
+ <td class="table-cell p-2">
+ <%= case key do %>
+ <% :query -> %>
+ <pre class="whitespace-pre-wrap"><%= value %></pre>
+ <% "params" -> %>
+ <pre><%= Jason.encode!(value, pretty: true) %></pre>
+ <% _ -> %>
+ <%= value %>
+ <% end %>
+ </td>
+ </tr>
+ <% end %>
+ </tbody>
+ </table>
+ </details>
+ <% end %>
+ </section>
+
+ <%= if length(@queries) == 0 do %>
+ <p class="text-gray-500">
+ No queries run by user in the last few minutes. Try opening a dashboard
+ </p>
+ <% end %>
+</div>
diff --git a/lib/plausible_web/views/debug_view.ex b/lib/plausible_web/views/debug_view.ex
new file mode 100644
index 0000000000..4e8ce74077
--- /dev/null
+++ b/lib/plausible_web/views/debug_view.ex
@@ -0,0 +1,11 @@
+defmodule PlausibleWeb.DebugView do
+ use PlausibleWeb, :view
+
+ def controller_name(phoenix_controller_name) do
+ phoenix_controller_name
+ |> String.to_existing_atom()
+ |> Module.split()
+ |> Enum.drop_while(&String.starts_with?(&1, "Plausible"))
+ |> Enum.join(".")
+ end
+end
diff --git a/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs b/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs
index 3c535d3cae..0cc2f76b7f 100644
--- a/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs
+++ b/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs
@@ -6,7 +6,7 @@ defmodule Plausible.IngestRepo.Migrations.AddImportedCustomEvents do
on_cluster = Plausible.MigrationUtils.on_cluster_statement("imported_pages")
settings =
- if Plausible.MigrationUtils.clustered_table?("imported_pages") do
+ if Plausible.IngestRepo.clustered_table?("imported_pages") do
"""
ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/{database}/imported_custom_events', '{replica}')
ORDER BY (site_id, import_id, date, name)