Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Add support for imported custom events (#4033)
Adrian Gruntkowski4e7e932a7519
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 86048662a8..1fe27d7b88 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -40,6 +40,8 @@ All notable changes to this project will be documented in this file.- Add `EXTRA_CONFIG_PATH` env var to specify extra Elixir config plausible/analytics#3906- Add restrictive `robots.txt` for self-hosted plausible/analytics#3905- Add Yesterday as an time range option in the dashboard+- Add support for importing Google Analytics 4 data+- Import custom events from Google Analytics 4### Removed- Removed the nested custom event property breakdown UI when filtering by a goal in Goal Conversionsdiff --git a/fixture/ga4_report_imported_custom_events.json b/fixture/ga4_report_imported_custom_events.jsonnew file mode 100644index 0000000000..c116736766--- /dev/null+++ b/fixture/ga4_report_imported_custom_events.json@@ -0,0 +1,1212 @@+{+ "kind": "analyticsData#batchRunReports",+ "reports": [+ {+ "dimensionHeaders": [+ {+ "name": "date"+ },+ {+ "name": "eventName"+ },+ {+ "name": "linkUrl"+ }+ ],+ "kind": "analyticsData#runReport",+ "metadata": {+ "currencyCode": "USD",+ "timeZone": "Europe/Amsterdam"+ },+ "metricHeaders": [+ {+ "name": "totalUsers",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "eventCount",+ "type": "TYPE_INTEGER"+ }+ ],+ "rowCount": 56,+ "rows": [+ {+ "dimensionValues": [+ {+ "value": "20240131"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "40"+ },+ {+ "value": "55"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240130"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "42"+ },+ {+ "value": "61"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240129"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "40"+ },+ {+ "value": "57"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240128"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "67"+ },+ {+ "value": "85"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240128"+ },+ {+ "value": "click"+ },+ {+ "value": "https://www.facebook.com/kuhinjskeprice"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240128"+ },+ {+ "value": "view_search_results"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240127"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "51"+ },+ {+ "value": "65"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240127"+ },+ {+ "value": "click"+ },+ {+ "value": "http://www.coolinarika.com/recept/omiljeni-mramorni-kuglof/"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240126"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "50"+ },+ {+ "value": "70"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240126"+ },+ {+ "value": "view_search_results"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240125"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "58"+ },+ {+ "value": "98"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240125"+ },+ {+ "value": "click"+ },+ {+ "value": "https://www.google.co.uk/imgres?imgurl=http://30.media.tumblr.com/oapJteChehjbp5traCrAENnXo1_500.jpg"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240125"+ },+ {+ "value": "view_search_results"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240124"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "61"+ },+ {+ "value": "97"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240124"+ },+ {+ "value": "click"+ },+ {+ "value": "http://www.ikea.com/dk/da/catalog/products/20213835/"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240124"+ },+ {+ "value": "click"+ },+ {+ "value": "https://www.facebook.com/kuhinjskeprice"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240123"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "74"+ },+ {+ "value": "114"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240123"+ },+ {+ "value": "view_search_results"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240122"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "68"+ },+ {+ "value": "95"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240122"+ },+ {+ "value": "click"+ },+ {+ "value": "https://pinterest.com/pin/68539225549526297/"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240122"+ },+ {+ "value": "click"+ },+ {+ "value": "https://www.facebook.com/kuhinjskeprice"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240121"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "116"+ },+ {+ "value": "181"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240121"+ },+ {+ "value": "view_search_results"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "2"+ },+ {+ "value": "3"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240120"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "111"+ },+ {+ "value": "190"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240120"+ },+ {+ "value": "click"+ },+ {+ "value": "https://www.facebook.com/kuhinjskeprice"+ }+ ],+ "metricValues": [+ {+ "value": "2"+ },+ {+ "value": "2"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240120"+ },+ {+ "value": "view_search_results"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "2"+ },+ {+ "value": "6"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240119"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "119"+ },+ {+ "value": "186"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240119"+ },+ {+ "value": "click"+ },+ {+ "value": "https://www.facebook.com/kuhinjskeprice"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240119"+ },+ {+ "value": "view_search_results"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240118"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "86"+ },+ {+ "value": "125"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240117"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "43"+ },+ {+ "value": "55"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240116"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "66"+ },+ {+ "value": "95"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240116"+ },+ {+ "value": "view_search_results"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240115"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "29"+ },+ {+ "value": "32"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240114"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "31"+ },+ {+ "value": "36"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240113"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "30"+ },+ {+ "value": "34"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240113"+ },+ {+ "value": "click"+ },+ {+ "value": "http://www.coolinarika.com/recept/omiljeni-mramorni-kuglof/"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240113"+ },+ {+ "value": "click"+ },+ {+ "value": "http://www.jamieoliver.com/recipes/pasta-recipes/spinach-ricotta-cannelloni/"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240112"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "31"+ },+ {+ "value": "33"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240112"+ },+ {+ "value": "click"+ },+ {+ "value": "http://www.ikea.com/dk/da/catalog/products/80010222/"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240111"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "26"+ },+ {+ "value": "29"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240111"+ },+ {+ "value": "click"+ },+ {+ "value": "http://www.apartmenttherapy.com/minh-and-ted-bought-a-distress-150686"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240110"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "24"+ },+ {+ "value": "28"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240109"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "32"+ },+ {+ "value": "56"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240108"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "24"+ },+ {+ "value": "26"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240108"+ },+ {+ "value": "click"+ },+ {+ "value": "http://www.coolinarika.com/recept/omiljeni-mramorni-kuglof/"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240107"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "27"+ },+ {+ "value": "45"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240107"+ },+ {+ "value": "view_search_results"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "15"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240106"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "22"+ },+ {+ "value": "22"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240105"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "38"+ },+ {+ "value": "39"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240104"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "24"+ },+ {+ "value": "27"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240103"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "34"+ },+ {+ "value": "35"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240103"+ },+ {+ "value": "click"+ },+ {+ "value": "https://pinterest.com/pin/238198267760892524/"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240102"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "29"+ },+ {+ "value": "39"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240102"+ },+ {+ "value": "click"+ },+ {+ "value": "http://www.lonelyplanet.com/denmark/images/legoland-copenhagen$25938-2"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240101"+ },+ {+ "value": "scroll"+ },+ {+ "value": ""+ }+ ],+ "metricValues": [+ {+ "value": "20"+ },+ {+ "value": "20"+ }+ ]+ }+ ]+ }+ ]+}\ No newline at end of filediff --git a/lib/plausible/google/ga4/http.ex b/lib/plausible/google/ga4/http.exindex 615b02bdc1..39eb97d423 100644--- a/lib/plausible/google/ga4/http.ex+++ b/lib/plausible/google/ga4/http.ex@@ -31,6 +31,7 @@ defmodule Plausible.Google.GA4.HTTP dodesc: true}],+ dimensionFilter: report_request.dimension_filter,limit: report_request.limit,offset: report_request.offset}diff --git a/lib/plausible/google/ga4/report_request.ex b/lib/plausible/google/ga4/report_request.exindex 5647ae19a1..7a94d8b0f3 100644--- a/lib/plausible/google/ga4/report_request.ex+++ b/lib/plausible/google/ga4/report_request.ex@@ -3,6 +3,13 @@ defmodule Plausible.Google.GA4.ReportRequest doReport request struct for Google Analytics 4 API"""+ @excluded_event_names [+ "page_view",+ "session_start",+ "first_visit",+ "user_engagement"+ ]+defstruct [:dataset,:dimensions,@@ -11,6 +18,7 @@ defmodule Plausible.Google.GA4.ReportRequest do:property,:access_token,:offset,+ :dimension_filter,:limit]@@ -91,6 +99,25 @@ defmodule Plausible.Google.GA4.ReportRequest do# "bounces = sessions - engagedSessions"# ]# },+ %__MODULE__{+ dataset: "imported_custom_events",+ dimensions: ["date", "eventName", "linkUrl"],+ metrics: [+ "totalUsers",+ "eventCount"+ ],+ dimension_filter: %{+ "notExpression" => %{+ "filter" => %{+ "fieldName" => "eventName",+ "inListFilter" => %{+ "values" => @excluded_event_names,+ "caseSensitive" => true+ }+ }+ }+ }+ },%__MODULE__{dataset: "imported_locations",dimensions: ["date", "countryId", "region", "city"],diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.exindex 3a6f8543e9..c914314a3e 100644--- a/lib/plausible/imported.ex+++ b/lib/plausible/imported.ex@@ -27,6 +27,7 @@ defmodule Plausible.Imported doImported.Page,Imported.EntryPage,Imported.ExitPage,+ Imported.CustomEvent,Imported.Location,Imported.Device,Imported.Browser,diff --git a/lib/plausible/imported/buffer.ex b/lib/plausible/imported/buffer.exindex fccbb62784..a1c50131a3 100644--- a/lib/plausible/imported/buffer.ex+++ b/lib/plausible/imported/buffer.ex@@ -107,6 +107,7 @@ defmodule Plausible.Imported.Buffer dodefp table_schema("imported_pages"), do: Plausible.Imported.Pagedefp table_schema("imported_entry_pages"), do: Plausible.Imported.EntryPagedefp table_schema("imported_exit_pages"), do: Plausible.Imported.ExitPage+ defp table_schema("imported_custom_events"), do: Plausible.Imported.CustomEventdefp table_schema("imported_locations"), do: Plausible.Imported.Locationdefp table_schema("imported_devices"), do: Plausible.Imported.Devicedefp table_schema("imported_browsers"), do: Plausible.Imported.Browserdiff --git a/lib/plausible/imported/custom_event.ex b/lib/plausible/imported/custom_event.exnew file mode 100644index 0000000000..c52b9fcfcb--- /dev/null+++ b/lib/plausible/imported/custom_event.ex@@ -0,0 +1,15 @@+defmodule Plausible.Imported.CustomEvent do+ @moduledoc false+ use Ecto.Schema++ @primary_key false+ schema "imported_custom_events" do+ field :site_id, Ch, type: "UInt64"+ field :import_id, Ch, type: "UInt64"+ field :date, :date+ field :name, :string+ field :link_url, :string+ field :visitors, Ch, type: "UInt64"+ field :events, Ch, type: "UInt64"+ end+enddiff --git a/lib/plausible/imported/google_analytics4.ex b/lib/plausible/imported/google_analytics4.exindex 0b4670124c..0b72c7a3f9 100644--- a/lib/plausible/imported/google_analytics4.ex+++ b/lib/plausible/imported/google_analytics4.ex@@ -149,6 +149,10 @@ defmodule Plausible.Imported.GoogleAnalytics4 doround(float)end+ defp maybe_override_event_name("file_download"), do: "File Download"+ defp maybe_override_event_name("click"), do: "Outbound Link: Click"+ defp maybe_override_event_name(name), do: name+defp new_from_report(site_id, import_id, "imported_visitors", row) do%{site_id: site_id,@@ -229,6 +233,18 @@ defmodule Plausible.Imported.GoogleAnalytics4 do# }# end+ defp new_from_report(site_id, import_id, "imported_custom_events", row) do+ %{+ site_id: site_id,+ import_id: import_id,+ date: get_date(row),+ name: row.dimensions |> Map.fetch!("eventName") |> maybe_override_event_name(),+ link_url: row.dimensions |> Map.fetch!("linkUrl"),+ visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),+ events: row.metrics |> Map.fetch!("eventCount") |> parse_number()+ }+ end+defp new_from_report(site_id, import_id, "imported_locations", row) docountry_code = row.dimensions |> Map.fetch!("countryId") |> default_if_missing("")city_name = row.dimensions |> Map.fetch!("city") |> default_if_missing("")diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex e1055e420f..991e80431e 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -45,7 +45,7 @@ defmodule Plausible.Stats.Aggregate dodefp aggregate_events(site, query, metrics) dofrom(e in base_event_query(site, query), select: ^select_event_metrics(metrics))|> merge_imported(site, query, metrics)- |> maybe_add_conversion_rate(site, query, metrics, include_imported: query.include_imported)+ |> maybe_add_conversion_rate(site, query, metrics)|> ClickhouseRepo.one()enddiff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex 298437880c..fe74fb426b 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -577,7 +577,7 @@ defmodule Plausible.Stats.Base do|> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id))end- defp total_visitors_subquery(site, query, true) do+ defp total_visitors_subquery(site, %Query{include_imported: true} = query) dodynamic([e],selected_as(@@ -588,16 +588,14 @@ defmodule Plausible.Stats.Base do)end- defp total_visitors_subquery(site, query, false) do+ defp total_visitors_subquery(site, query) dodynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors))enddef add_percentage_metric(q, site, query, metrics) doif :percentage in metrics doq- |> select_merge(- ^%{__total_visitors: total_visitors_subquery(site, query, query.include_imported)}- )+ |> select_merge(^%{__total_visitors: total_visitors_subquery(site, query)})|> select_merge(%{percentage:fragment(@@ -615,17 +613,13 @@ defmodule Plausible.Stats.Base do# Adds conversion_rate metric to query, calculated as# X / Y where Y is the same breakdown value without goal or props# filters.- def maybe_add_conversion_rate(q, site, query, metrics, opts) do+ def maybe_add_conversion_rate(q, site, query, metrics) doif :conversion_rate in metrics do- include_imported = Keyword.fetch!(opts, :include_imported)-total_query = query |> Query.remove_event_filters([:goal, :props])# :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALLsubquery(q)- |> select_merge(- ^%{total_visitors: total_visitors_subquery(site, total_query, include_imported)}- )+ |> select_merge(^%{total_visitors: total_visitors_subquery(site, total_query)})|> select_merge([e], %{conversion_rate:fragment(diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 7512df1a3f..566da8a071 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -94,6 +94,7 @@ defmodule Plausible.Stats.Breakdown do})|> select_merge(^select_columns)+ |> merge_imported_pageview_goals(site, query, page_exprs, metrics_to_select)|> apply_pagination(pagination)elsenil@@ -120,7 +121,7 @@ defmodule Plausible.Stats.Breakdown doif full_q dofull_q- |> maybe_add_conversion_rate(site, query, metrics, include_imported: false)+ |> maybe_add_conversion_rate(site, query, metrics)|> ClickhouseRepo.all()|> transform_keys(%{name: :goal})|> cast_revenue_metrics_to_money(revenue_goals)@@ -149,7 +150,7 @@ defmodule Plausible.Stats.Breakdown doif !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)breakdown_events(site, query, metrics_to_select)- |> maybe_add_conversion_rate(site, query, metrics, include_imported: false)+ |> maybe_add_conversion_rate(site, query, metrics)|> paginate_and_execute(metrics, pagination)|> transform_keys(%{breakdown_prop_value: custom_prop})|> Enum.map(&cast_revenue_metrics_to_money(&1, currency))diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.exindex 5906a5858b..9ab3dd703e 100644--- a/lib/plausible/stats/imported.ex+++ b/lib/plausible/stats/imported.ex@@ -1,6 +1,6 @@defmodule Plausible.Stats.Imported douse Plausible.ClickhouseRepo- alias Plausible.Stats.Query+ alias Plausible.Stats.{Query, Base}import Ecto.Queryimport Plausible.Stats.Fragments@@ -26,11 +26,34 @@ defmodule Plausible.Stats.Imported do"visit:browser_version" => "imported_browsers","visit:os" => "imported_operating_systems","visit:os_version" => "imported_operating_systems",- "event:page" => "imported_pages"+ "event:page" => "imported_pages",+ "event:name" => "imported_custom_events",+ "event:props:url" => "imported_custom_events"}@imported_properties Map.keys(@property_to_table_mappings)+ def schema_supports_query?(query) do+ filter_count = length(Map.keys(query.filters))++ case {filter_count, query.property} do+ {0, "event:props:" <> _} -> false+ {0, _} -> true+ {1, _} -> supports_single_filter?(query)+ {_, _} -> false+ end+ end++ defp supports_single_filter?(%Query{+ filters: %{"event:goal" => {:is, {:event, event}}},+ property: "event:props:url"+ })+ when event in ["Outbound Link: Click", "File Download"] do+ true+ end++ defp supports_single_filter?(_query), do: false+def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{include_imported: false}, _),do: native_q@@ -79,7 +102,6 @@ defmodule Plausible.Stats.Imported doenddef merge_imported(q, _, %Query{include_imported: false}, _), do: q- def merge_imported(q, _, _, [:events | _]), do: qdef merge_imported(q, site, %Query{property: property} = query, metrics)when property in @imported_properties do@@ -96,12 +118,15 @@ defmodule Plausible.Stats.Imported dowhere: i.visitors > 0,select: %{})- |> maybe_filter_by_breakdown_property(query.filters[property], dim)+ |> maybe_apply_filter(query.filters, property, dim)|> group_imported_by(dim)|> select_imported_metrics(metrics)join_on =case dim do+ :url ->+ dynamic([s, i], s.breakdown_prop_value == i.breakdown_prop_value)+:os_version ->dynamic([s, i], s.os == i.os and s.os_version == i.os_version)@@ -137,6 +162,41 @@ defmodule Plausible.Stats.Imported dodef merge_imported(q, _, _, _), do: q+ def merge_imported_pageview_goals(q, _, %Query{include_imported: false}, _, _), do: q++ def merge_imported_pageview_goals(q, site, query, page_exprs, metrics) do+ page_regexes = Enum.map(page_exprs, &Base.page_regex/1)++ imported_q =+ from(+ i in "imported_pages",+ where: i.site_id == ^site.id,+ where: i.import_id in ^site.complete_import_ids,+ where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,+ where: i.visitors > 0,+ where:+ fragment(+ "notEmpty(multiMatchAllIndices(?, ?) as indices)",+ i.page,+ ^page_regexes+ ),+ array_join: index in fragment("indices"),+ group_by: index,+ select: %{+ name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)+ }+ )+ |> select_imported_metrics(metrics)++ from(s in Ecto.Query.subquery(q),+ full_join: i in subquery(imported_q),+ on: s.name == i.name,+ select: %{}+ )+ |> select_joined_dimension(:name)+ |> select_joined_metrics(metrics)+ end+def total_imported_visitors(site, query) doimported_visitors(site, query)|> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)})@@ -154,11 +214,22 @@ defmodule Plausible.Stats.Imported do)end- defp maybe_filter_by_breakdown_property(q, {:member, list}, dim) do- where(q, [i], field(i, ^dim) in ^list)+ defp maybe_apply_filter(+ q,+ %{"event:goal" => {:is, {:event, event_name}}},+ "event:props:url",+ _dim+ )+ when event_name in ["Outbound Link: Click", "File Download"] do+ where(q, [i], i.name == ^event_name)end- defp maybe_filter_by_breakdown_property(q, _, _), do: q+ defp maybe_apply_filter(q, filters, property, dim) do+ case filters[property] do+ {:member, list} -> where(q, [i], field(i, ^dim) in ^list)+ _ -> q+ end+ enddefp select_imported_metrics(q, []), do: q@@ -168,6 +239,21 @@ defmodule Plausible.Stats.Imported do|> select_imported_metrics(rest)end+ defp select_imported_metrics(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q,+ [:events | rest]+ ) do+ q+ |> select_merge([i], %{events: sum(i.events)})+ |> select_imported_metrics(rest)+ end++ defp select_imported_metrics(q, [:events | rest]) do+ q+ |> select_merge([i], %{events: sum(i.pageviews)})+ |> select_imported_metrics(rest)+ end+defp select_imported_metrics(%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,[:visits | rest]@@ -192,6 +278,15 @@ defmodule Plausible.Stats.Imported do|> select_imported_metrics(rest)end+ defp select_imported_metrics(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q,+ [:pageviews | rest]+ ) do+ q+ |> select_merge([i], %{pageviews: 0})+ |> select_imported_metrics(rest)+ end+defp select_imported_metrics(q, [:pageviews | rest]) doq|> where([i], i.pageviews > 0)@@ -351,6 +446,18 @@ defmodule Plausible.Stats.Imported do|> select_merge([i], %{^dim => field(i, ^dim)})end+ defp group_imported_by(q, :name) do+ q+ |> group_by([i], i.name)+ |> select_merge([i], %{name: i.name})+ end++ defp group_imported_by(q, :url) do+ q+ |> group_by([i], i.link_url)+ |> select_merge([i], %{breakdown_prop_value: i.link_url})+ end+defp select_joined_dimension(q, :city) doselect_merge(q, [s, i], %{city: fragment("greatest(?,?)", i.city, s.city)@@ -372,6 +479,18 @@ defmodule Plausible.Stats.Imported do})end+ defp select_joined_dimension(q, :url) do+ select_merge(q, [s, i], %{+ breakdown_prop_value:+ fragment(+ "if(empty(?), ?, ?)",+ s.breakdown_prop_value,+ i.breakdown_prop_value,+ s.breakdown_prop_value+ )+ })+ end+defp select_joined_dimension(q, dim) doselect_merge(q, [s, i], %{^dim => fragment("if(empty(?), ?, ?)", field(s, ^dim), field(i, ^dim), field(s, ^dim))@@ -397,6 +516,12 @@ defmodule Plausible.Stats.Imported do|> select_joined_metrics(rest)end+ defp select_joined_metrics(q, [:events | rest]) do+ q+ |> select_merge([s, i], %{events: s.events + i.events})+ |> select_joined_metrics(rest)+ end+defp select_joined_metrics(q, [:pageviews | rest]) doq|> select_merge([s, i], %{pageviews: s.pageviews + i.pageviews})diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex 00bf937ed8..5af5ede7f4 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -14,7 +14,7 @@ defmodule Plausible.Stats.Query doexperimental_reduced_joins?: falserequire OpenTelemetry.Tracer, as: Tracer- alias Plausible.Stats.{Filters, Interval}+ alias Plausible.Stats.{Filters, Interval, Imported}@type t :: %__MODULE__{}@@ -273,7 +273,7 @@ defmodule Plausible.Stats.Query docond dois_nil(site.latest_import_end_date) -> falseDate.after?(query.date_range.first, site.latest_import_end_date) -> false- Enum.any?(query.filters) -> false+ not Imported.schema_supports_query?(query) -> falsequery.period == "realtime" -> falsetrue -> requested?enddiff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.exindex 3d2df1cb33..0623873993 100644--- a/lib/plausible_web/controllers/api/external_stats_controller.ex+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex@@ -192,15 +192,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController doendend- defp validate_metric("events" = metric, query) do- if query.include_imported do- {:error, "Metric `#{metric}` cannot be queried with imported data"}- else- {:ok, metric}- end- end-- defp validate_metric(metric, _) when metric in ["visitors", "pageviews"] do+ defp validate_metric(metric, _) when metric in ["visitors", "pageviews", "events"] do{:ok, metric}enddiff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex a36b3f6791..66659f3e5d 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -1204,9 +1204,7 @@ defmodule PlausibleWeb.Api.StatsController doparams = Map.put(params, "property", prefixed_prop)- query =- Query.from(site, params)- |> Map.put(:include_imported, false)+ query = Query.from(site, params)metrics =if query.filters["event:goal"] dodiff --git a/lib/workers/clickhouse_clean_sites.ex b/lib/workers/clickhouse_clean_sites.exindex 0d2475d064..fbe67ee6f0 100644--- a/lib/workers/clickhouse_clean_sites.ex+++ b/lib/workers/clickhouse_clean_sites.ex@@ -26,6 +26,7 @@ defmodule Plausible.Workers.ClickhouseCleanSites do"imported_locations","imported_operating_systems","imported_pages",+ "imported_custom_events","imported_sources","imported_visitors"]diff --git a/test/plausible/imported/google_analytics4_test.exs b/test/plausible/imported/google_analytics4_test.exsindex 84ca050956..d89912c1da 100644--- a/test/plausible/imported/google_analytics4_test.exs+++ b/test/plausible/imported/google_analytics4_test.exs@@ -16,6 +16,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do"fixture/ga4_report_imported_sources.json","fixture/ga4_report_imported_pages.json","fixture/ga4_report_imported_entry_pages.json",+ "fixture/ga4_report_imported_custom_events.json","fixture/ga4_report_imported_locations.json","fixture/ga4_report_imported_devices.json","fixture/ga4_report_imported_browsers.json",@@ -89,6 +90,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do"imported_pages" -> 3340"imported_entry_pages" -> 2934"imported_exit_pages" -> 0+ "imported_custom_events" -> 56"imported_locations" -> 2291"imported_devices" -> 93"imported_browsers" -> 233@@ -122,9 +124,19 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doconn = put_req_header(conn, "authorization", "Bearer #{api_key}")+ insert(:goal, event_name: "Outbound Link: Click", site: site)+ insert(:goal, event_name: "view_search_results", site: site)+ insert(:goal, event_name: "scroll", site: site)++ # Timeseriesassert_timeseries(conn, common_params)++ # Breakdown (event:*)assert_pages(conn, common_params)+ assert_custom_events(conn, common_params)+ assert_outbound_link_urls(conn, common_params)+ # Breakdown (visit:*)assert_sources(conn, breakdown_params)assert_utm_mediums(conn, breakdown_params)assert_entry_pages(conn, breakdown_params)@@ -133,6 +145,8 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert_browsers(conn, breakdown_params)assert_os(conn, breakdown_params)assert_os_versions(conn, breakdown_params)++ # Miscassert_active_visitors(site_import)end@@ -202,6 +216,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do"imported_devices" -> 0"imported_browsers" -> 0"imported_operating_systems" -> 0+ "imported_custom_events" -> 0endquery = from(imported in table, where: imported.site_id == ^site.id)@@ -229,6 +244,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do"imported_devices" -> 93"imported_browsers" -> 233"imported_operating_systems" -> 1068+ "imported_custom_events" -> 56endquery = from(imported in table, where: imported.site_id == ^site.id)@@ -265,6 +281,67 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doend)end+ defp assert_custom_events(conn, params) do+ params =+ params+ |> Map.put("metrics", "visitors,events,conversion_rate")+ |> Map.put("property", "event:goal")++ %{"results" => results} =+ get(conn, "/api/v1/stats/breakdown", params) |> json_response(200)++ assert results == [+ %{+ "goal" => "scroll",+ "visitors" => 1513,+ "events" => 2130,+ "conversion_rate" => 24.7+ },+ %{+ "goal" => "Outbound Link: Click",+ "visitors" => 17,+ "events" => 17,+ "conversion_rate" => 0.3+ },+ %{+ "goal" => "view_search_results",+ "visitors" => 11,+ "events" => 30,+ "conversion_rate" => 0.2+ }+ ]+ end++ defp assert_outbound_link_urls(conn, params) do+ params =+ Map.merge(params, %{+ "metrics" => "visitors,events,conversion_rate",+ "property" => "event:props:url",+ "filters" => "event:goal==Outbound Link: Click"+ })++ %{"results" => results} =+ get(conn, "/api/v1/stats/breakdown", params) |> json_response(200)++ assert length(results) == 10++ assert List.first(results) ==+ %{+ "url" => "https://www.facebook.com/kuhinjskeprice",+ "visitors" => 6,+ "conversion_rate" => 0.1,+ "events" => 6+ }++ assert %{+ "url" =>+ "http://www.jamieoliver.com/recipes/pasta-recipes/spinach-ricotta-cannelloni/",+ "visitors" => 1,+ "conversion_rate" => 0.0,+ "events" => 1+ } in results+ end+defp assert_timeseries(conn, params) doparams =Map.put(diff --git a/test/plausible/purge_test.exs b/test/plausible/purge_test.exsindex a06ba9061f..14cbbc7836 100644--- a/test/plausible/purge_test.exs+++ b/test/plausible/purge_test.exs@@ -30,6 +30,7 @@ defmodule Plausible.PurgeTest dobuild(:imported_pages),build(:imported_entry_pages),build(:imported_exit_pages),+ build(:imported_custom_events),build(:imported_locations),build(:imported_devices),build(:imported_browsers),@@ -43,6 +44,7 @@ defmodule Plausible.PurgeTest dobuild(:imported_pages),build(:imported_entry_pages),build(:imported_exit_pages),+ build(:imported_custom_events),build(:imported_locations),build(:imported_devices),build(:imported_browsers),@@ -101,6 +103,7 @@ defmodule Plausible.PurgeTest dobuild(:imported_pages),build(:imported_entry_pages),build(:imported_exit_pages),+ build(:imported_custom_events),build(:imported_locations),build(:imported_devices),build(:imported_browsers),diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exsindex 3e38c748e3..f568589549 100644--- a/test/plausible/stats/query_test.exs+++ b/test/plausible/stats/query_test.exs@@ -204,4 +204,79 @@ defmodule Plausible.Stats.QueryTest doassert q.filters["visit:source"] == {:is, "Twitter"}endend++ describe "include_imported" do+ setup [:create_site]++ test "is true when requested via params and imported data exists", %{site: site} do+ insert(:site_import, site: site)+ site = Plausible.Imported.load_import_data(site)++ assert %{include_imported: true} =+ Query.from(site, %{"period" => "day", "with_imported" => "true"})+ end++ test "is false when imported data does not exist", %{site: site} do+ assert %{include_imported: false} =+ Query.from(site, %{"period" => "day", "with_imported" => "true"})+ end++ test "is false when imported data exists but is out of the date range", %{site: site} do+ insert(:site_import, site: site, start_date: ~D[2021-01-01], end_date: ~D[2022-01-01])+ site = Plausible.Imported.load_import_data(site)++ assert %{include_imported: false} =+ Query.from(site, %{"period" => "day", "with_imported" => "true"})+ end++ test "is false in realtime even when imported data from today exists", %{site: site} do+ insert(:site_import, site: site)+ site = Plausible.Imported.load_import_data(site)++ assert %{include_imported: false} =+ Query.from(site, %{"period" => "realtime", "with_imported" => "true"})+ end++ test "is false when an arbitrary custom property filter is used", %{site: site} do+ insert(:site_import, site: site)+ site = Plausible.Imported.load_import_data(site)++ assert %{include_imported: false} =+ Query.from(site, %{+ "period" => "day",+ "with_imported" => "true",+ "property" => "event:props:url",+ "filters" => Jason.encode!(%{"props" => %{"author" => "!John Doe"}})+ })+ end++ test "is true when breaking down by url and filtering by outbound link or file download goal",+ %{site: site} do+ insert(:site_import, site: site)+ site = Plausible.Imported.load_import_data(site)++ Enum.each(["Outbound Link: Click", "File Download"], fn goal_name ->+ assert %{include_imported: true} =+ Query.from(site, %{+ "period" => "day",+ "with_imported" => "true",+ "property" => "event:props:url",+ "filters" => Jason.encode!(%{"goal" => goal_name})+ })+ end)+ end++ test "is false when breaking down by url but without a special goal filter",+ %{site: site} do+ insert(:site_import, site: site)+ site = Plausible.Imported.load_import_data(site)++ assert %{include_imported: false} =+ Query.from(site, %{+ "period" => "day",+ "with_imported" => "true",+ "property" => "event:props:url"+ })+ end+ endenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exsindex 8561eb3d81..cb248f1f66 100644--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs@@ -588,28 +588,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do"visitors" => %{"value" => 1, "change" => 100}}end-- test "events metric with imported data is disallowed", %{- conn: conn,- site: site,- site_import: site_import- } do- populate_stats(site, site_import.id, [- build(:imported_visitors, date: ~D[2023-01-01])- ])-- conn =- get(conn, "/api/v1/stats/aggregate", %{- "site_id" => site.domain,- "period" => "day",- "date" => "2023-01-02",- "metrics" => "events",- "with_imported" => "true"- })-- assert %{"error" => msg} = json_response(conn, 400)- assert msg == "Metric `events` cannot be queried with imported data"- endenddescribe "filters" dodiff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exsindex 178c7e9328..0b1ae85040 100644--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs@@ -3053,4 +3053,173 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do}endend++ describe "imported data" do+ test "returns custom event goals and pageview goals", %{conn: conn, site: site} do+ insert(:goal, site: site, event_name: "Purchase")+ insert(:goal, site: site, page_path: "/test")++ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview,+ timestamp: ~N[2021-01-01 00:00:01],+ pathname: "/test"+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ ),+ build(:imported_custom_events,+ name: "Purchase",+ visitors: 3,+ events: 5,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages,+ page: "/test",+ visitors: 2,+ pageviews: 2,+ date: ~D[2021-01-01]+ ),+ build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])+ ])++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "date" => "2021-01-01",+ "property" => "event:goal",+ "metrics" => "visitors,events,pageviews,conversion_rate",+ "with_imported" => "true"+ })++ assert [+ %{+ "goal" => "Purchase",+ "visitors" => 5,+ "events" => 7,+ "pageviews" => 0,+ "conversion_rate" => 62.5+ },+ %{+ "goal" => "Visit /test",+ "visitors" => 3,+ "events" => 3,+ "pageviews" => 3,+ "conversion_rate" => 37.5+ }+ ] = json_response(conn, 200)["results"]+ end++ test "pageviews are returned as events for breakdown reports other than custom events", %{+ conn: conn,+ site: site+ } do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:imported_browsers, browser: "Chrome", pageviews: 1, date: ~D[2021-01-01]),+ build(:imported_devices, device: "Desktop", pageviews: 1, date: ~D[2021-01-01]),+ build(:imported_entry_pages, entry_page: "/test", pageviews: 1, date: ~D[2021-01-01]),+ build(:imported_exit_pages, exit_page: "/test", pageviews: 1, date: ~D[2021-01-01]),+ build(:imported_locations, country: "EE", pageviews: 1, date: ~D[2021-01-01]),+ build(:imported_operating_systems,+ operating_system: "Mac",+ pageviews: 1,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages, page: "/test", pageviews: 1, date: ~D[2021-01-01]),+ build(:imported_sources, source: "Google", pageviews: 1, date: ~D[2021-01-01])+ ])++ params = %{+ "site_id" => site.domain,+ "period" => "day",+ "date" => "2021-01-01",+ "metrics" => "events",+ "with_imported" => "true"+ }++ breakdown_and_first = fn property ->+ conn+ |> get("/api/v1/stats/breakdown", Map.put(params, "property", property))+ |> json_response(200)+ |> Map.get("results")+ |> List.first()+ end++ assert %{"browser" => "Chrome", "events" => 1} = breakdown_and_first.("visit:browser")+ assert %{"device" => "Desktop", "events" => 1} = breakdown_and_first.("visit:device")+ assert %{"entry_page" => "/test", "events" => 1} = breakdown_and_first.("visit:entry_page")+ assert %{"exit_page" => "/test", "events" => 1} = breakdown_and_first.("visit:exit_page")+ assert %{"country" => "EE", "events" => 1} = breakdown_and_first.("visit:country")+ assert %{"os" => "Mac", "events" => 1} = breakdown_and_first.("visit:os")+ assert %{"page" => "/test", "events" => 1} = breakdown_and_first.("event:page")+ assert %{"source" => "Google", "events" => 1} = breakdown_and_first.("visit:source")+ end++ for goal_name <- ["Outbound Link: Click", "File Download"] do+ test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do+ insert(:goal, event_name: unquote(goal_name), site: site)+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:event,+ name: unquote(goal_name),+ "meta.key": ["url"],+ "meta.value": ["https://one.com"]+ ),+ build(:imported_custom_events,+ name: unquote(goal_name),+ visitors: 2,+ events: 5,+ link_url: "https://one.com"+ ),+ build(:imported_custom_events,+ name: unquote(goal_name),+ visitors: 5,+ events: 10,+ link_url: "https://two.com"+ ),+ build(:imported_custom_events,+ name: "some goal",+ visitors: 5,+ events: 10+ ),+ build(:imported_visitors, visitors: 9)+ ])++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "property" => "event:props:url",+ "filters" => "event:goal==#{unquote(goal_name)}",+ "metrics" => "visitors,events,conversion_rate",+ "with_imported" => "true"+ })++ assert json_response(conn, 200)["results"] == [+ %{+ "visitors" => 5,+ "url" => "https://two.com",+ "events" => 10,+ "conversion_rate" => 50.0+ },+ %{+ "visitors" => 3,+ "url" => "https://one.com",+ "events" => 6,+ "conversion_rate" => 30.0+ }+ ]+ end+ end+ endenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exsindex f9d6186b49..75bc9655ef 100644--- a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs@@ -1525,4 +1525,32 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do}endend++ describe "imported data" do+ test "returns pageviews as the value of events metric", %{+ conn: conn,+ site: site+ } do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:imported_visitors, pageviews: 1, date: ~D[2021-01-01])+ ])++ first_result =+ conn+ |> get("/api/v1/stats/timeseries", %{+ "site_id" => site.domain,+ "period" => "7d",+ "metrics" => "events",+ "date" => "2021-01-07",+ "with_imported" => "true"+ })+ |> json_response(200)+ |> Map.get("results")+ |> List.first()++ assert first_result == %{"date" => "2021-01-01", "events" => 1}+ end+ endenddiff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exsindex 5d8e9e9c5a..f932456dfb 100644--- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs@@ -626,44 +626,6 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do}]end-- test "conversion_rate for goals should not be calculated with imported data", %{- conn: conn,- site: site- } do- site_import =- insert(:site_import,- start_date: ~D[2005-01-01],- end_date: Timex.today(),- source: :universal_analytics- )-- populate_stats(site, site_import.id, [- build(:pageview, pathname: "/"),- build(:pageview, pathname: "/another"),- build(:pageview, pathname: "/blog/post-1"),- build(:pageview, pathname: "/blog/post-2"),- build(:imported_pages, page: "/blog/post-1"),- build(:imported_visitors)- ])-- insert(:goal, %{site: site, page_path: "/blog**"})-- conn =- get(- conn,- "/api/stats/#{site.domain}/conversions?period=day"- )-- assert json_response(conn, 200) == [- %{- "name" => "Visit /blog**",- "visitors" => 2,- "events" => 2,- "conversion_rate" => 50- }- ]- endenddescribe "GET /api/stats/:domain/conversions - with goal and prop=(none) filter" do@@ -785,4 +747,99 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do]endend++ describe "GET /api/stats/:domain/conversions - with imported data" do+ setup [:create_user, :log_in, :create_site]++ test "returns custom event goals and pageview goals", %{conn: conn, site: site} do+ insert(:goal, site: site, event_name: "Purchase")+ insert(:goal, site: site, page_path: "/test")++ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview,+ timestamp: ~N[2021-01-01 00:00:01],+ pathname: "/test"+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ ),+ build(:imported_custom_events,+ name: "Purchase",+ visitors: 3,+ events: 5,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages,+ page: "/test",+ visitors: 2,+ pageviews: 2,+ date: ~D[2021-01-01]+ ),+ build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])+ ])++ url_query_params = "?period=day&date=2021-01-01&with_imported=true"+ conn = get(conn, "/api/stats/#{site.domain}/conversions#{url_query_params}")++ assert [+ %{+ "name" => "Purchase",+ "visitors" => 5,+ "events" => 7,+ "conversion_rate" => 62.5+ },+ %{+ "name" => "Visit /test",+ "visitors" => 3,+ "events" => 3,+ "conversion_rate" => 37.5+ }+ ] = json_response(conn, 200)+ end++ test "calculates conversion_rate for goals with glob pattern with imported data", %{+ conn: conn,+ site: site+ } do+ site_import =+ insert(:site_import,+ start_date: ~D[2005-01-01],+ end_date: Timex.today(),+ source: :universal_analytics+ )++ populate_stats(site, site_import.id, [+ build(:pageview, pathname: "/"),+ build(:pageview, pathname: "/another"),+ build(:pageview, pathname: "/blog/post-1"),+ build(:pageview, pathname: "/blog/post-2"),+ build(:imported_pages, page: "/blog/post-1"),+ build(:imported_visitors)+ ])++ insert(:goal, %{site: site, page_path: "/blog**"})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/conversions?period=day"+ )++ assert json_response(conn, 200) == [+ %{+ "name" => "Visit /blog**",+ "visitors" => 2,+ "events" => 2,+ "conversion_rate" => 50+ }+ ]+ end+ endenddiff --git a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exsindex d53658806c..9c1898a02c 100644--- a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs@@ -1132,4 +1132,64 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do}endend++ describe "with imported data" do+ setup [:create_user, :log_in, :create_new_site]++ for goal_name <- ["Outbound Link: Click", "File Download"] do+ test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do+ insert(:goal, event_name: unquote(goal_name), site: site)+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:event,+ name: unquote(goal_name),+ "meta.key": ["url"],+ "meta.value": ["https://one.com"]+ ),+ build(:imported_custom_events,+ name: unquote(goal_name),+ visitors: 2,+ events: 5,+ link_url: "https://one.com"+ ),+ build(:imported_custom_events,+ name: unquote(goal_name),+ visitors: 5,+ events: 10,+ link_url: "https://two.com"+ ),+ build(:imported_custom_events,+ name: "view_search_results",+ visitors: 100,+ events: 200+ ),+ build(:imported_visitors, visitors: 9)+ ])++ filters = Jason.encode!(%{goal: unquote(goal_name)})++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/custom-prop-values/url?period=day&with_imported=true&filters=#{filters}"+ )++ assert json_response(conn, 200) == [+ %{+ "visitors" => 5,+ "name" => "https://two.com",+ "events" => 10,+ "conversion_rate" => 50.0+ },+ %{+ "visitors" => 3,+ "name" => "https://one.com",+ "events" => 6,+ "conversion_rate" => 30.0+ }+ ]+ end+ end+ endenddiff --git a/test/support/factory.ex b/test/support/factory.exindex 6ab32a7501..d2856ed21d 100644--- a/test/support/factory.ex+++ b/test/support/factory.ex@@ -237,6 +237,17 @@ defmodule Plausible.Factory do}end+ def imported_custom_events_factory do+ %{+ table: "imported_custom_events",+ date: Timex.today(),+ name: "",+ link_url: "",+ visitors: 1,+ events: 1+ }+ end+def imported_locations_factory do%{table: "imported_locations",