Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Add support for imported custom events (#4033)

Adrian Gruntkowski4e7e932a7519
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 86048662a8..1fe27d7b88 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -40,6 +40,8 @@ All notable changes to this project will be documented in this file.
- Add `EXTRA_CONFIG_PATH` env var to specify extra Elixir config plausible/analytics#3906
- Add restrictive `robots.txt` for self-hosted plausible/analytics#3905
- Add Yesterday as an time range option in the dashboard
+- Add support for importing Google Analytics 4 data
+- Import custom events from Google Analytics 4
### Removed
- Removed the nested custom event property breakdown UI when filtering by a goal in Goal Conversions
diff --git a/fixture/ga4_report_imported_custom_events.json b/fixture/ga4_report_imported_custom_events.json
new file mode 100644
index 0000000000..c116736766
--- /dev/null
+++ b/fixture/ga4_report_imported_custom_events.json
@@ -0,0 +1,1212 @@
+{
+ "kind": "analyticsData#batchRunReports",
+ "reports": [
+ {
+ "dimensionHeaders": [
+ {
+ "name": "date"
+ },
+ {
+ "name": "eventName"
+ },
+ {
+ "name": "linkUrl"
+ }
+ ],
+ "kind": "analyticsData#runReport",
+ "metadata": {
+ "currencyCode": "USD",
+ "timeZone": "Europe/Amsterdam"
+ },
+ "metricHeaders": [
+ {
+ "name": "totalUsers",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "eventCount",
+ "type": "TYPE_INTEGER"
+ }
+ ],
+ "rowCount": 56,
+ "rows": [
+ {
+ "dimensionValues": [
+ {
+ "value": "20240131"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "40"
+ },
+ {
+ "value": "55"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240130"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "42"
+ },
+ {
+ "value": "61"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240129"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "40"
+ },
+ {
+ "value": "57"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240128"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "67"
+ },
+ {
+ "value": "85"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240128"
+ },
+ {
+ "value": "click"
+ },
+ {
+ "value": "https://www.facebook.com/kuhinjskeprice"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240128"
+ },
+ {
+ "value": "view_search_results"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240127"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "51"
+ },
+ {
+ "value": "65"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240127"
+ },
+ {
+ "value": "click"
+ },
+ {
+ "value": "http://www.coolinarika.com/recept/omiljeni-mramorni-kuglof/"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240126"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "50"
+ },
+ {
+ "value": "70"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240126"
+ },
+ {
+ "value": "view_search_results"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240125"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "58"
+ },
+ {
+ "value": "98"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240125"
+ },
+ {
+ "value": "click"
+ },
+ {
+ "value": "https://www.google.co.uk/imgres?imgurl=http://30.media.tumblr.com/oapJteChehjbp5traCrAENnXo1_500.jpg"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240125"
+ },
+ {
+ "value": "view_search_results"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240124"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "61"
+ },
+ {
+ "value": "97"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240124"
+ },
+ {
+ "value": "click"
+ },
+ {
+ "value": "http://www.ikea.com/dk/da/catalog/products/20213835/"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240124"
+ },
+ {
+ "value": "click"
+ },
+ {
+ "value": "https://www.facebook.com/kuhinjskeprice"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240123"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "74"
+ },
+ {
+ "value": "114"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240123"
+ },
+ {
+ "value": "view_search_results"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240122"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "68"
+ },
+ {
+ "value": "95"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240122"
+ },
+ {
+ "value": "click"
+ },
+ {
+ "value": "https://pinterest.com/pin/68539225549526297/"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240122"
+ },
+ {
+ "value": "click"
+ },
+ {
+ "value": "https://www.facebook.com/kuhinjskeprice"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240121"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "116"
+ },
+ {
+ "value": "181"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240121"
+ },
+ {
+ "value": "view_search_results"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "2"
+ },
+ {
+ "value": "3"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240120"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "111"
+ },
+ {
+ "value": "190"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240120"
+ },
+ {
+ "value": "click"
+ },
+ {
+ "value": "https://www.facebook.com/kuhinjskeprice"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "2"
+ },
+ {
+ "value": "2"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240120"
+ },
+ {
+ "value": "view_search_results"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "2"
+ },
+ {
+ "value": "6"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240119"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "119"
+ },
+ {
+ "value": "186"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240119"
+ },
+ {
+ "value": "click"
+ },
+ {
+ "value": "https://www.facebook.com/kuhinjskeprice"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240119"
+ },
+ {
+ "value": "view_search_results"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240118"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "86"
+ },
+ {
+ "value": "125"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240117"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "43"
+ },
+ {
+ "value": "55"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240116"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "66"
+ },
+ {
+ "value": "95"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240116"
+ },
+ {
+ "value": "view_search_results"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240115"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "29"
+ },
+ {
+ "value": "32"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240114"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "31"
+ },
+ {
+ "value": "36"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240113"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "30"
+ },
+ {
+ "value": "34"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240113"
+ },
+ {
+ "value": "click"
+ },
+ {
+ "value": "http://www.coolinarika.com/recept/omiljeni-mramorni-kuglof/"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240113"
+ },
+ {
+ "value": "click"
+ },
+ {
+ "value": "http://www.jamieoliver.com/recipes/pasta-recipes/spinach-ricotta-cannelloni/"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240112"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "31"
+ },
+ {
+ "value": "33"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240112"
+ },
+ {
+ "value": "click"
+ },
+ {
+ "value": "http://www.ikea.com/dk/da/catalog/products/80010222/"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240111"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "26"
+ },
+ {
+ "value": "29"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240111"
+ },
+ {
+ "value": "click"
+ },
+ {
+ "value": "http://www.apartmenttherapy.com/minh-and-ted-bought-a-distress-150686"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240110"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "24"
+ },
+ {
+ "value": "28"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240109"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "32"
+ },
+ {
+ "value": "56"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240108"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "24"
+ },
+ {
+ "value": "26"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240108"
+ },
+ {
+ "value": "click"
+ },
+ {
+ "value": "http://www.coolinarika.com/recept/omiljeni-mramorni-kuglof/"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240107"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "27"
+ },
+ {
+ "value": "45"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240107"
+ },
+ {
+ "value": "view_search_results"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "15"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240106"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "22"
+ },
+ {
+ "value": "22"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240105"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "38"
+ },
+ {
+ "value": "39"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240104"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "24"
+ },
+ {
+ "value": "27"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240103"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "34"
+ },
+ {
+ "value": "35"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240103"
+ },
+ {
+ "value": "click"
+ },
+ {
+ "value": "https://pinterest.com/pin/238198267760892524/"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240102"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "29"
+ },
+ {
+ "value": "39"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240102"
+ },
+ {
+ "value": "click"
+ },
+ {
+ "value": "http://www.lonelyplanet.com/denmark/images/legoland-copenhagen$25938-2"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240101"
+ },
+ {
+ "value": "scroll"
+ },
+ {
+ "value": ""
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "20"
+ },
+ {
+ "value": "20"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/lib/plausible/google/ga4/http.ex b/lib/plausible/google/ga4/http.ex
index 615b02bdc1..39eb97d423 100644
--- a/lib/plausible/google/ga4/http.ex
+++ b/lib/plausible/google/ga4/http.ex
@@ -31,6 +31,7 @@ defmodule Plausible.Google.GA4.HTTP do
desc: true
}
],
+ dimensionFilter: report_request.dimension_filter,
limit: report_request.limit,
offset: report_request.offset
}
diff --git a/lib/plausible/google/ga4/report_request.ex b/lib/plausible/google/ga4/report_request.ex
index 5647ae19a1..7a94d8b0f3 100644
--- a/lib/plausible/google/ga4/report_request.ex
+++ b/lib/plausible/google/ga4/report_request.ex
@@ -3,6 +3,13 @@ defmodule Plausible.Google.GA4.ReportRequest do
Report request struct for Google Analytics 4 API
"""
+ @excluded_event_names [
+ "page_view",
+ "session_start",
+ "first_visit",
+ "user_engagement"
+ ]
+
defstruct [
:dataset,
:dimensions,
@@ -11,6 +18,7 @@ defmodule Plausible.Google.GA4.ReportRequest do
:property,
:access_token,
:offset,
+ :dimension_filter,
:limit
]
@@ -91,6 +99,25 @@ defmodule Plausible.Google.GA4.ReportRequest do
# "bounces = sessions - engagedSessions"
# ]
# },
+ %__MODULE__{
+ dataset: "imported_custom_events",
+ dimensions: ["date", "eventName", "linkUrl"],
+ metrics: [
+ "totalUsers",
+ "eventCount"
+ ],
+ dimension_filter: %{
+ "notExpression" => %{
+ "filter" => %{
+ "fieldName" => "eventName",
+ "inListFilter" => %{
+ "values" => @excluded_event_names,
+ "caseSensitive" => true
+ }
+ }
+ }
+ }
+ },
%__MODULE__{
dataset: "imported_locations",
dimensions: ["date", "countryId", "region", "city"],
diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.ex
index 3a6f8543e9..c914314a3e 100644
--- a/lib/plausible/imported.ex
+++ b/lib/plausible/imported.ex
@@ -27,6 +27,7 @@ defmodule Plausible.Imported do
Imported.Page,
Imported.EntryPage,
Imported.ExitPage,
+ Imported.CustomEvent,
Imported.Location,
Imported.Device,
Imported.Browser,
diff --git a/lib/plausible/imported/buffer.ex b/lib/plausible/imported/buffer.ex
index fccbb62784..a1c50131a3 100644
--- a/lib/plausible/imported/buffer.ex
+++ b/lib/plausible/imported/buffer.ex
@@ -107,6 +107,7 @@ defmodule Plausible.Imported.Buffer do
defp table_schema("imported_pages"), do: Plausible.Imported.Page
defp table_schema("imported_entry_pages"), do: Plausible.Imported.EntryPage
defp table_schema("imported_exit_pages"), do: Plausible.Imported.ExitPage
+ defp table_schema("imported_custom_events"), do: Plausible.Imported.CustomEvent
defp table_schema("imported_locations"), do: Plausible.Imported.Location
defp table_schema("imported_devices"), do: Plausible.Imported.Device
defp table_schema("imported_browsers"), do: Plausible.Imported.Browser
diff --git a/lib/plausible/imported/custom_event.ex b/lib/plausible/imported/custom_event.ex
new file mode 100644
index 0000000000..c52b9fcfcb
--- /dev/null
+++ b/lib/plausible/imported/custom_event.ex
@@ -0,0 +1,15 @@
+defmodule Plausible.Imported.CustomEvent do
+ @moduledoc false
+ use Ecto.Schema
+
+ @primary_key false
+ schema "imported_custom_events" do
+ field :site_id, Ch, type: "UInt64"
+ field :import_id, Ch, type: "UInt64"
+ field :date, :date
+ field :name, :string
+ field :link_url, :string
+ field :visitors, Ch, type: "UInt64"
+ field :events, Ch, type: "UInt64"
+ end
+end
diff --git a/lib/plausible/imported/google_analytics4.ex b/lib/plausible/imported/google_analytics4.ex
index 0b4670124c..0b72c7a3f9 100644
--- a/lib/plausible/imported/google_analytics4.ex
+++ b/lib/plausible/imported/google_analytics4.ex
@@ -149,6 +149,10 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
round(float)
end
+ defp maybe_override_event_name("file_download"), do: "File Download"
+ defp maybe_override_event_name("click"), do: "Outbound Link: Click"
+ defp maybe_override_event_name(name), do: name
+
defp new_from_report(site_id, import_id, "imported_visitors", row) do
%{
site_id: site_id,
@@ -229,6 +233,18 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
# }
# end
+ defp new_from_report(site_id, import_id, "imported_custom_events", row) do
+ %{
+ site_id: site_id,
+ import_id: import_id,
+ date: get_date(row),
+ name: row.dimensions |> Map.fetch!("eventName") |> maybe_override_event_name(),
+ link_url: row.dimensions |> Map.fetch!("linkUrl"),
+ visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+ events: row.metrics |> Map.fetch!("eventCount") |> parse_number()
+ }
+ end
+
defp new_from_report(site_id, import_id, "imported_locations", row) do
country_code = row.dimensions |> Map.fetch!("countryId") |> default_if_missing("")
city_name = row.dimensions |> Map.fetch!("city") |> default_if_missing("")
diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex
index e1055e420f..991e80431e 100644
--- a/lib/plausible/stats/aggregate.ex
+++ b/lib/plausible/stats/aggregate.ex
@@ -45,7 +45,7 @@ defmodule Plausible.Stats.Aggregate do
defp aggregate_events(site, query, metrics) do
from(e in base_event_query(site, query), select: ^select_event_metrics(metrics))
|> merge_imported(site, query, metrics)
- |> maybe_add_conversion_rate(site, query, metrics, include_imported: query.include_imported)
+ |> maybe_add_conversion_rate(site, query, metrics)
|> ClickhouseRepo.one()
end
diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex
index 298437880c..fe74fb426b 100644
--- a/lib/plausible/stats/base.ex
+++ b/lib/plausible/stats/base.ex
@@ -577,7 +577,7 @@ defmodule Plausible.Stats.Base do
|> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id))
end
- defp total_visitors_subquery(site, query, true) do
+ defp total_visitors_subquery(site, %Query{include_imported: true} = query) do
dynamic(
[e],
selected_as(
@@ -588,16 +588,14 @@ defmodule Plausible.Stats.Base do
)
end
- defp total_visitors_subquery(site, query, false) do
+ defp total_visitors_subquery(site, query) do
dynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors))
end
def add_percentage_metric(q, site, query, metrics) do
if :percentage in metrics do
q
- |> select_merge(
- ^%{__total_visitors: total_visitors_subquery(site, query, query.include_imported)}
- )
+ |> select_merge(^%{__total_visitors: total_visitors_subquery(site, query)})
|> select_merge(%{
percentage:
fragment(
@@ -615,17 +613,13 @@ defmodule Plausible.Stats.Base do
# Adds conversion_rate metric to query, calculated as
# X / Y where Y is the same breakdown value without goal or props
# filters.
- def maybe_add_conversion_rate(q, site, query, metrics, opts) do
+ def maybe_add_conversion_rate(q, site, query, metrics) do
if :conversion_rate in metrics do
- include_imported = Keyword.fetch!(opts, :include_imported)
-
total_query = query |> Query.remove_event_filters([:goal, :props])
# :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL
subquery(q)
- |> select_merge(
- ^%{total_visitors: total_visitors_subquery(site, total_query, include_imported)}
- )
+ |> select_merge(^%{total_visitors: total_visitors_subquery(site, total_query)})
|> select_merge([e], %{
conversion_rate:
fragment(
diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index 7512df1a3f..566da8a071 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -94,6 +94,7 @@ defmodule Plausible.Stats.Breakdown do
}
)
|> select_merge(^select_columns)
+ |> merge_imported_pageview_goals(site, query, page_exprs, metrics_to_select)
|> apply_pagination(pagination)
else
nil
@@ -120,7 +121,7 @@ defmodule Plausible.Stats.Breakdown do
if full_q do
full_q
- |> maybe_add_conversion_rate(site, query, metrics, include_imported: false)
+ |> maybe_add_conversion_rate(site, query, metrics)
|> ClickhouseRepo.all()
|> transform_keys(%{name: :goal})
|> cast_revenue_metrics_to_money(revenue_goals)
@@ -149,7 +150,7 @@ defmodule Plausible.Stats.Breakdown do
if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)
breakdown_events(site, query, metrics_to_select)
- |> maybe_add_conversion_rate(site, query, metrics, include_imported: false)
+ |> maybe_add_conversion_rate(site, query, metrics)
|> paginate_and_execute(metrics, pagination)
|> transform_keys(%{breakdown_prop_value: custom_prop})
|> Enum.map(&cast_revenue_metrics_to_money(&1, currency))
diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex
index 5906a5858b..9ab3dd703e 100644
--- a/lib/plausible/stats/imported.ex
+++ b/lib/plausible/stats/imported.ex
@@ -1,6 +1,6 @@
defmodule Plausible.Stats.Imported do
use Plausible.ClickhouseRepo
- alias Plausible.Stats.Query
+ alias Plausible.Stats.{Query, Base}
import Ecto.Query
import Plausible.Stats.Fragments
@@ -26,11 +26,34 @@ defmodule Plausible.Stats.Imported do
"visit:browser_version" => "imported_browsers",
"visit:os" => "imported_operating_systems",
"visit:os_version" => "imported_operating_systems",
- "event:page" => "imported_pages"
+ "event:page" => "imported_pages",
+ "event:name" => "imported_custom_events",
+ "event:props:url" => "imported_custom_events"
}
@imported_properties Map.keys(@property_to_table_mappings)
+ def schema_supports_query?(query) do
+ filter_count = length(Map.keys(query.filters))
+
+ case {filter_count, query.property} do
+ {0, "event:props:" <> _} -> false
+ {0, _} -> true
+ {1, _} -> supports_single_filter?(query)
+ {_, _} -> false
+ end
+ end
+
+ defp supports_single_filter?(%Query{
+ filters: %{"event:goal" => {:is, {:event, event}}},
+ property: "event:props:url"
+ })
+ when event in ["Outbound Link: Click", "File Download"] do
+ true
+ end
+
+ defp supports_single_filter?(_query), do: false
+
def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{include_imported: false}, _),
do: native_q
@@ -79,7 +102,6 @@ defmodule Plausible.Stats.Imported do
end
def merge_imported(q, _, %Query{include_imported: false}, _), do: q
- def merge_imported(q, _, _, [:events | _]), do: q
def merge_imported(q, site, %Query{property: property} = query, metrics)
when property in @imported_properties do
@@ -96,12 +118,15 @@ defmodule Plausible.Stats.Imported do
where: i.visitors > 0,
select: %{}
)
- |> maybe_filter_by_breakdown_property(query.filters[property], dim)
+ |> maybe_apply_filter(query.filters, property, dim)
|> group_imported_by(dim)
|> select_imported_metrics(metrics)
join_on =
case dim do
+ :url ->
+ dynamic([s, i], s.breakdown_prop_value == i.breakdown_prop_value)
+
:os_version ->
dynamic([s, i], s.os == i.os and s.os_version == i.os_version)
@@ -137,6 +162,41 @@ defmodule Plausible.Stats.Imported do
def merge_imported(q, _, _, _), do: q
+ def merge_imported_pageview_goals(q, _, %Query{include_imported: false}, _, _), do: q
+
+ def merge_imported_pageview_goals(q, site, query, page_exprs, metrics) do
+ page_regexes = Enum.map(page_exprs, &Base.page_regex/1)
+
+ imported_q =
+ from(
+ i in "imported_pages",
+ where: i.site_id == ^site.id,
+ where: i.import_id in ^site.complete_import_ids,
+ where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,
+ where: i.visitors > 0,
+ where:
+ fragment(
+ "notEmpty(multiMatchAllIndices(?, ?) as indices)",
+ i.page,
+ ^page_regexes
+ ),
+ array_join: index in fragment("indices"),
+ group_by: index,
+ select: %{
+ name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)
+ }
+ )
+ |> select_imported_metrics(metrics)
+
+ from(s in Ecto.Query.subquery(q),
+ full_join: i in subquery(imported_q),
+ on: s.name == i.name,
+ select: %{}
+ )
+ |> select_joined_dimension(:name)
+ |> select_joined_metrics(metrics)
+ end
+
def total_imported_visitors(site, query) do
imported_visitors(site, query)
|> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)})
@@ -154,11 +214,22 @@ defmodule Plausible.Stats.Imported do
)
end
- defp maybe_filter_by_breakdown_property(q, {:member, list}, dim) do
- where(q, [i], field(i, ^dim) in ^list)
+ defp maybe_apply_filter(
+ q,
+ %{"event:goal" => {:is, {:event, event_name}}},
+ "event:props:url",
+ _dim
+ )
+ when event_name in ["Outbound Link: Click", "File Download"] do
+ where(q, [i], i.name == ^event_name)
end
- defp maybe_filter_by_breakdown_property(q, _, _), do: q
+ defp maybe_apply_filter(q, filters, property, dim) do
+ case filters[property] do
+ {:member, list} -> where(q, [i], field(i, ^dim) in ^list)
+ _ -> q
+ end
+ end
defp select_imported_metrics(q, []), do: q
@@ -168,6 +239,21 @@ defmodule Plausible.Stats.Imported do
|> select_imported_metrics(rest)
end
+ defp select_imported_metrics(
+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q,
+ [:events | rest]
+ ) do
+ q
+ |> select_merge([i], %{events: sum(i.events)})
+ |> select_imported_metrics(rest)
+ end
+
+ defp select_imported_metrics(q, [:events | rest]) do
+ q
+ |> select_merge([i], %{events: sum(i.pageviews)})
+ |> select_imported_metrics(rest)
+ end
+
defp select_imported_metrics(
%Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_exit_pages", _}}} = q,
[:visits | rest]
@@ -192,6 +278,15 @@ defmodule Plausible.Stats.Imported do
|> select_imported_metrics(rest)
end
+ defp select_imported_metrics(
+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"imported_custom_events", _}}} = q,
+ [:pageviews | rest]
+ ) do
+ q
+ |> select_merge([i], %{pageviews: 0})
+ |> select_imported_metrics(rest)
+ end
+
defp select_imported_metrics(q, [:pageviews | rest]) do
q
|> where([i], i.pageviews > 0)
@@ -351,6 +446,18 @@ defmodule Plausible.Stats.Imported do
|> select_merge([i], %{^dim => field(i, ^dim)})
end
+ defp group_imported_by(q, :name) do
+ q
+ |> group_by([i], i.name)
+ |> select_merge([i], %{name: i.name})
+ end
+
+ defp group_imported_by(q, :url) do
+ q
+ |> group_by([i], i.link_url)
+ |> select_merge([i], %{breakdown_prop_value: i.link_url})
+ end
+
defp select_joined_dimension(q, :city) do
select_merge(q, [s, i], %{
city: fragment("greatest(?,?)", i.city, s.city)
@@ -372,6 +479,18 @@ defmodule Plausible.Stats.Imported do
})
end
+ defp select_joined_dimension(q, :url) do
+ select_merge(q, [s, i], %{
+ breakdown_prop_value:
+ fragment(
+ "if(empty(?), ?, ?)",
+ s.breakdown_prop_value,
+ i.breakdown_prop_value,
+ s.breakdown_prop_value
+ )
+ })
+ end
+
defp select_joined_dimension(q, dim) do
select_merge(q, [s, i], %{
^dim => fragment("if(empty(?), ?, ?)", field(s, ^dim), field(i, ^dim), field(s, ^dim))
@@ -397,6 +516,12 @@ defmodule Plausible.Stats.Imported do
|> select_joined_metrics(rest)
end
+ defp select_joined_metrics(q, [:events | rest]) do
+ q
+ |> select_merge([s, i], %{events: s.events + i.events})
+ |> select_joined_metrics(rest)
+ end
+
defp select_joined_metrics(q, [:pageviews | rest]) do
q
|> select_merge([s, i], %{pageviews: s.pageviews + i.pageviews})
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index 00bf937ed8..5af5ede7f4 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -14,7 +14,7 @@ defmodule Plausible.Stats.Query do
experimental_reduced_joins?: false
require OpenTelemetry.Tracer, as: Tracer
- alias Plausible.Stats.{Filters, Interval}
+ alias Plausible.Stats.{Filters, Interval, Imported}
@type t :: %__MODULE__{}
@@ -273,7 +273,7 @@ defmodule Plausible.Stats.Query do
cond do
is_nil(site.latest_import_end_date) -> false
Date.after?(query.date_range.first, site.latest_import_end_date) -> false
- Enum.any?(query.filters) -> false
+ not Imported.schema_supports_query?(query) -> false
query.period == "realtime" -> false
true -> requested?
end
diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex
index 3d2df1cb33..0623873993 100644
--- a/lib/plausible_web/controllers/api/external_stats_controller.ex
+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex
@@ -192,15 +192,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
end
end
- defp validate_metric("events" = metric, query) do
- if query.include_imported do
- {:error, "Metric `#{metric}` cannot be queried with imported data"}
- else
- {:ok, metric}
- end
- end
-
- defp validate_metric(metric, _) when metric in ["visitors", "pageviews"] do
+ defp validate_metric(metric, _) when metric in ["visitors", "pageviews", "events"] do
{:ok, metric}
end
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index a36b3f6791..66659f3e5d 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -1204,9 +1204,7 @@ defmodule PlausibleWeb.Api.StatsController do
params = Map.put(params, "property", prefixed_prop)
- query =
- Query.from(site, params)
- |> Map.put(:include_imported, false)
+ query = Query.from(site, params)
metrics =
if query.filters["event:goal"] do
diff --git a/lib/workers/clickhouse_clean_sites.ex b/lib/workers/clickhouse_clean_sites.ex
index 0d2475d064..fbe67ee6f0 100644
--- a/lib/workers/clickhouse_clean_sites.ex
+++ b/lib/workers/clickhouse_clean_sites.ex
@@ -26,6 +26,7 @@ defmodule Plausible.Workers.ClickhouseCleanSites do
"imported_locations",
"imported_operating_systems",
"imported_pages",
+ "imported_custom_events",
"imported_sources",
"imported_visitors"
]
diff --git a/test/plausible/imported/google_analytics4_test.exs b/test/plausible/imported/google_analytics4_test.exs
index 84ca050956..d89912c1da 100644
--- a/test/plausible/imported/google_analytics4_test.exs
+++ b/test/plausible/imported/google_analytics4_test.exs
@@ -16,6 +16,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
"fixture/ga4_report_imported_sources.json",
"fixture/ga4_report_imported_pages.json",
"fixture/ga4_report_imported_entry_pages.json",
+ "fixture/ga4_report_imported_custom_events.json",
"fixture/ga4_report_imported_locations.json",
"fixture/ga4_report_imported_devices.json",
"fixture/ga4_report_imported_browsers.json",
@@ -89,6 +90,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
"imported_pages" -> 3340
"imported_entry_pages" -> 2934
"imported_exit_pages" -> 0
+ "imported_custom_events" -> 56
"imported_locations" -> 2291
"imported_devices" -> 93
"imported_browsers" -> 233
@@ -122,9 +124,19 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
conn = put_req_header(conn, "authorization", "Bearer #{api_key}")
+ insert(:goal, event_name: "Outbound Link: Click", site: site)
+ insert(:goal, event_name: "view_search_results", site: site)
+ insert(:goal, event_name: "scroll", site: site)
+
+ # Timeseries
assert_timeseries(conn, common_params)
+
+ # Breakdown (event:*)
assert_pages(conn, common_params)
+ assert_custom_events(conn, common_params)
+ assert_outbound_link_urls(conn, common_params)
+ # Breakdown (visit:*)
assert_sources(conn, breakdown_params)
assert_utm_mediums(conn, breakdown_params)
assert_entry_pages(conn, breakdown_params)
@@ -133,6 +145,8 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert_browsers(conn, breakdown_params)
assert_os(conn, breakdown_params)
assert_os_versions(conn, breakdown_params)
+
+ # Misc
assert_active_visitors(site_import)
end
@@ -202,6 +216,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
"imported_devices" -> 0
"imported_browsers" -> 0
"imported_operating_systems" -> 0
+ "imported_custom_events" -> 0
end
query = from(imported in table, where: imported.site_id == ^site.id)
@@ -229,6 +244,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
"imported_devices" -> 93
"imported_browsers" -> 233
"imported_operating_systems" -> 1068
+ "imported_custom_events" -> 56
end
query = from(imported in table, where: imported.site_id == ^site.id)
@@ -265,6 +281,67 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
end)
end
+ defp assert_custom_events(conn, params) do
+ params =
+ params
+ |> Map.put("metrics", "visitors,events,conversion_rate")
+ |> Map.put("property", "event:goal")
+
+ %{"results" => results} =
+ get(conn, "/api/v1/stats/breakdown", params) |> json_response(200)
+
+ assert results == [
+ %{
+ "goal" => "scroll",
+ "visitors" => 1513,
+ "events" => 2130,
+ "conversion_rate" => 24.7
+ },
+ %{
+ "goal" => "Outbound Link: Click",
+ "visitors" => 17,
+ "events" => 17,
+ "conversion_rate" => 0.3
+ },
+ %{
+ "goal" => "view_search_results",
+ "visitors" => 11,
+ "events" => 30,
+ "conversion_rate" => 0.2
+ }
+ ]
+ end
+
+ defp assert_outbound_link_urls(conn, params) do
+ params =
+ Map.merge(params, %{
+ "metrics" => "visitors,events,conversion_rate",
+ "property" => "event:props:url",
+ "filters" => "event:goal==Outbound Link: Click"
+ })
+
+ %{"results" => results} =
+ get(conn, "/api/v1/stats/breakdown", params) |> json_response(200)
+
+ assert length(results) == 10
+
+ assert List.first(results) ==
+ %{
+ "url" => "https://www.facebook.com/kuhinjskeprice",
+ "visitors" => 6,
+ "conversion_rate" => 0.1,
+ "events" => 6
+ }
+
+ assert %{
+ "url" =>
+ "http://www.jamieoliver.com/recipes/pasta-recipes/spinach-ricotta-cannelloni/",
+ "visitors" => 1,
+ "conversion_rate" => 0.0,
+ "events" => 1
+ } in results
+ end
+
defp assert_timeseries(conn, params) do
params =
Map.put(
diff --git a/test/plausible/purge_test.exs b/test/plausible/purge_test.exs
index a06ba9061f..14cbbc7836 100644
--- a/test/plausible/purge_test.exs
+++ b/test/plausible/purge_test.exs
@@ -30,6 +30,7 @@ defmodule Plausible.PurgeTest do
build(:imported_pages),
build(:imported_entry_pages),
build(:imported_exit_pages),
+ build(:imported_custom_events),
build(:imported_locations),
build(:imported_devices),
build(:imported_browsers),
@@ -43,6 +44,7 @@ defmodule Plausible.PurgeTest do
build(:imported_pages),
build(:imported_entry_pages),
build(:imported_exit_pages),
+ build(:imported_custom_events),
build(:imported_locations),
build(:imported_devices),
build(:imported_browsers),
@@ -101,6 +103,7 @@ defmodule Plausible.PurgeTest do
build(:imported_pages),
build(:imported_entry_pages),
build(:imported_exit_pages),
+ build(:imported_custom_events),
build(:imported_locations),
build(:imported_devices),
build(:imported_browsers),
diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs
index 3e38c748e3..f568589549 100644
--- a/test/plausible/stats/query_test.exs
+++ b/test/plausible/stats/query_test.exs
@@ -204,4 +204,79 @@ defmodule Plausible.Stats.QueryTest do
assert q.filters["visit:source"] == {:is, "Twitter"}
end
end
+
+ describe "include_imported" do
+ setup [:create_site]
+
+ test "is true when requested via params and imported data exists", %{site: site} do
+ insert(:site_import, site: site)
+ site = Plausible.Imported.load_import_data(site)
+
+ assert %{include_imported: true} =
+ Query.from(site, %{"period" => "day", "with_imported" => "true"})
+ end
+
+ test "is false when imported data does not exist", %{site: site} do
+ assert %{include_imported: false} =
+ Query.from(site, %{"period" => "day", "with_imported" => "true"})
+ end
+
+ test "is false when imported data exists but is out of the date range", %{site: site} do
+ insert(:site_import, site: site, start_date: ~D[2021-01-01], end_date: ~D[2022-01-01])
+ site = Plausible.Imported.load_import_data(site)
+
+ assert %{include_imported: false} =
+ Query.from(site, %{"period" => "day", "with_imported" => "true"})
+ end
+
+ test "is false in realtime even when imported data from today exists", %{site: site} do
+ insert(:site_import, site: site)
+ site = Plausible.Imported.load_import_data(site)
+
+ assert %{include_imported: false} =
+ Query.from(site, %{"period" => "realtime", "with_imported" => "true"})
+ end
+
+ test "is false when an arbitrary custom property filter is used", %{site: site} do
+ insert(:site_import, site: site)
+ site = Plausible.Imported.load_import_data(site)
+
+ assert %{include_imported: false} =
+ Query.from(site, %{
+ "period" => "day",
+ "with_imported" => "true",
+ "property" => "event:props:url",
+ "filters" => Jason.encode!(%{"props" => %{"author" => "!John Doe"}})
+ })
+ end
+
+ test "is true when breaking down by url and filtering by outbound link or file download goal",
+ %{site: site} do
+ insert(:site_import, site: site)
+ site = Plausible.Imported.load_import_data(site)
+
+ Enum.each(["Outbound Link: Click", "File Download"], fn goal_name ->
+ assert %{include_imported: true} =
+ Query.from(site, %{
+ "period" => "day",
+ "with_imported" => "true",
+ "property" => "event:props:url",
+ "filters" => Jason.encode!(%{"goal" => goal_name})
+ })
+ end)
+ end
+
+ test "is false when breaking down by url but without a special goal filter",
+ %{site: site} do
+ insert(:site_import, site: site)
+ site = Plausible.Imported.load_import_data(site)
+
+ assert %{include_imported: false} =
+ Query.from(site, %{
+ "period" => "day",
+ "with_imported" => "true",
+ "property" => "event:props:url"
+ })
+ end
+ end
end
diff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
index 8561eb3d81..cb248f1f66 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
@@ -588,28 +588,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
"visitors" => %{"value" => 1, "change" => 100}
}
end
-
- test "events metric with imported data is disallowed", %{
- conn: conn,
- site: site,
- site_import: site_import
- } do
- populate_stats(site, site_import.id, [
- build(:imported_visitors, date: ~D[2023-01-01])
- ])
-
- conn =
- get(conn, "/api/v1/stats/aggregate", %{
- "site_id" => site.domain,
- "period" => "day",
- "date" => "2023-01-02",
- "metrics" => "events",
- "with_imported" => "true"
- })
-
- assert %{"error" => msg} = json_response(conn, 400)
- assert msg == "Metric `events` cannot be queried with imported data"
- end
end
describe "filters" do
diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
index 178c7e9328..0b1ae85040 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
@@ -3053,4 +3053,173 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
}
end
end
+
+ describe "imported data" do
+ test "returns custom event goals and pageview goals", %{conn: conn, site: site} do
+ insert(:goal, site: site, event_name: "Purchase")
+ insert(:goal, site: site, page_path: "/test")
+
+ site_import = insert(:site_import, site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:pageview,
+ timestamp: ~N[2021-01-01 00:00:01],
+ pathname: "/test"
+ ),
+ build(:event,
+ name: "Purchase",
+ timestamp: ~N[2021-01-01 00:00:03]
+ ),
+ build(:event,
+ name: "Purchase",
+ timestamp: ~N[2021-01-01 00:00:03]
+ ),
+ build(:imported_custom_events,
+ name: "Purchase",
+ visitors: 3,
+ events: 5,
+ date: ~D[2021-01-01]
+ ),
+ build(:imported_pages,
+ page: "/test",
+ visitors: 2,
+ pageviews: 2,
+ date: ~D[2021-01-01]
+ ),
+ build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])
+ ])
+
+ conn =
+ get(conn, "/api/v1/stats/breakdown", %{
+ "site_id" => site.domain,
+ "period" => "day",
+ "date" => "2021-01-01",
+ "property" => "event:goal",
+ "metrics" => "visitors,events,pageviews,conversion_rate",
+ "with_imported" => "true"
+ })
+
+ assert [
+ %{
+ "goal" => "Purchase",
+ "visitors" => 5,
+ "events" => 7,
+ "pageviews" => 0,
+ "conversion_rate" => 62.5
+ },
+ %{
+ "goal" => "Visit /test",
+ "visitors" => 3,
+ "events" => 3,
+ "pageviews" => 3,
+ "conversion_rate" => 37.5
+ }
+ ] = json_response(conn, 200)["results"]
+ end
+
+ test "pageviews are returned as events for breakdown reports other than custom events", %{
+ conn: conn,
+ site: site
+ } do
+ site_import = insert(:site_import, site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:imported_browsers, browser: "Chrome", pageviews: 1, date: ~D[2021-01-01]),
+ build(:imported_devices, device: "Desktop", pageviews: 1, date: ~D[2021-01-01]),
+ build(:imported_entry_pages, entry_page: "/test", pageviews: 1, date: ~D[2021-01-01]),
+ build(:imported_exit_pages, exit_page: "/test", pageviews: 1, date: ~D[2021-01-01]),
+ build(:imported_locations, country: "EE", pageviews: 1, date: ~D[2021-01-01]),
+ build(:imported_operating_systems,
+ operating_system: "Mac",
+ pageviews: 1,
+ date: ~D[2021-01-01]
+ ),
+ build(:imported_pages, page: "/test", pageviews: 1, date: ~D[2021-01-01]),
+ build(:imported_sources, source: "Google", pageviews: 1, date: ~D[2021-01-01])
+ ])
+
+ params = %{
+ "site_id" => site.domain,
+ "period" => "day",
+ "date" => "2021-01-01",
+ "metrics" => "events",
+ "with_imported" => "true"
+ }
+
+ breakdown_and_first = fn property ->
+ conn
+ |> get("/api/v1/stats/breakdown", Map.put(params, "property", property))
+ |> json_response(200)
+ |> Map.get("results")
+ |> List.first()
+ end
+
+ assert %{"browser" => "Chrome", "events" => 1} = breakdown_and_first.("visit:browser")
+ assert %{"device" => "Desktop", "events" => 1} = breakdown_and_first.("visit:device")
+ assert %{"entry_page" => "/test", "events" => 1} = breakdown_and_first.("visit:entry_page")
+ assert %{"exit_page" => "/test", "events" => 1} = breakdown_and_first.("visit:exit_page")
+ assert %{"country" => "EE", "events" => 1} = breakdown_and_first.("visit:country")
+ assert %{"os" => "Mac", "events" => 1} = breakdown_and_first.("visit:os")
+ assert %{"page" => "/test", "events" => 1} = breakdown_and_first.("event:page")
+ assert %{"source" => "Google", "events" => 1} = breakdown_and_first.("visit:source")
+ end
+
+ for goal_name <- ["Outbound Link: Click", "File Download"] do
+ test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do
+ insert(:goal, event_name: unquote(goal_name), site: site)
+ site_import = insert(:site_import, site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:event,
+ name: unquote(goal_name),
+ "meta.key": ["url"],
+ "meta.value": ["https://one.com"]
+ ),
+ build(:imported_custom_events,
+ name: unquote(goal_name),
+ visitors: 2,
+ events: 5,
+ link_url: "https://one.com"
+ ),
+ build(:imported_custom_events,
+ name: unquote(goal_name),
+ visitors: 5,
+ events: 10,
+ link_url: "https://two.com"
+ ),
+ build(:imported_custom_events,
+ name: "some goal",
+ visitors: 5,
+ events: 10
+ ),
+ build(:imported_visitors, visitors: 9)
+ ])
+
+ conn =
+ get(conn, "/api/v1/stats/breakdown", %{
+ "site_id" => site.domain,
+ "period" => "day",
+ "property" => "event:props:url",
+ "filters" => "event:goal==#{unquote(goal_name)}",
+ "metrics" => "visitors,events,conversion_rate",
+ "with_imported" => "true"
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{
+ "visitors" => 5,
+ "url" => "https://two.com",
+ "events" => 10,
+ "conversion_rate" => 50.0
+ },
+ %{
+ "visitors" => 3,
+ "url" => "https://one.com",
+ "events" => 6,
+ "conversion_rate" => 30.0
+ }
+ ]
+ end
+ end
+ end
end
diff --git a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs
index f9d6186b49..75bc9655ef 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs
@@ -1525,4 +1525,32 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do
}
end
end
+
+ describe "imported data" do
+ test "returns pageviews as the value of events metric", %{
+ conn: conn,
+ site: site
+ } do
+ site_import = insert(:site_import, site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:imported_visitors, pageviews: 1, date: ~D[2021-01-01])
+ ])
+
+ first_result =
+ conn
+ |> get("/api/v1/stats/timeseries", %{
+ "site_id" => site.domain,
+ "period" => "7d",
+ "metrics" => "events",
+ "date" => "2021-01-07",
+ "with_imported" => "true"
+ })
+ |> json_response(200)
+ |> Map.get("results")
+ |> List.first()
+
+ assert first_result == %{"date" => "2021-01-01", "events" => 1}
+ end
+ end
end
diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
index 5d8e9e9c5a..f932456dfb 100644
--- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
@@ -626,44 +626,6 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
}
]
end
-
- test "conversion_rate for goals should not be calculated with imported data", %{
- conn: conn,
- site: site
- } do
- site_import =
- insert(:site_import,
- start_date: ~D[2005-01-01],
- end_date: Timex.today(),
- source: :universal_analytics
- )
-
- populate_stats(site, site_import.id, [
- build(:pageview, pathname: "/"),
- build(:pageview, pathname: "/another"),
- build(:pageview, pathname: "/blog/post-1"),
- build(:pageview, pathname: "/blog/post-2"),
- build(:imported_pages, page: "/blog/post-1"),
- build(:imported_visitors)
- ])
-
- insert(:goal, %{site: site, page_path: "/blog**"})
-
- conn =
- get(
- conn,
- "/api/stats/#{site.domain}/conversions?period=day"
- )
-
- assert json_response(conn, 200) == [
- %{
- "name" => "Visit /blog**",
- "visitors" => 2,
- "events" => 2,
- "conversion_rate" => 50
- }
- ]
- end
end
describe "GET /api/stats/:domain/conversions - with goal and prop=(none) filter" do
@@ -785,4 +747,99 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
]
end
end
+
+ describe "GET /api/stats/:domain/conversions - with imported data" do
+ setup [:create_user, :log_in, :create_site]
+
+ test "returns custom event goals and pageview goals", %{conn: conn, site: site} do
+ insert(:goal, site: site, event_name: "Purchase")
+ insert(:goal, site: site, page_path: "/test")
+
+ site_import = insert(:site_import, site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:pageview,
+ timestamp: ~N[2021-01-01 00:00:01],
+ pathname: "/test"
+ ),
+ build(:event,
+ name: "Purchase",
+ timestamp: ~N[2021-01-01 00:00:03]
+ ),
+ build(:event,
+ name: "Purchase",
+ timestamp: ~N[2021-01-01 00:00:03]
+ ),
+ build(:imported_custom_events,
+ name: "Purchase",
+ visitors: 3,
+ events: 5,
+ date: ~D[2021-01-01]
+ ),
+ build(:imported_pages,
+ page: "/test",
+ visitors: 2,
+ pageviews: 2,
+ date: ~D[2021-01-01]
+ ),
+ build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])
+ ])
+
+ url_query_params = "?period=day&date=2021-01-01&with_imported=true"
+ conn = get(conn, "/api/stats/#{site.domain}/conversions#{url_query_params}")
+
+ assert [
+ %{
+ "name" => "Purchase",
+ "visitors" => 5,
+ "events" => 7,
+ "conversion_rate" => 62.5
+ },
+ %{
+ "name" => "Visit /test",
+ "visitors" => 3,
+ "events" => 3,
+ "conversion_rate" => 37.5
+ }
+ ] = json_response(conn, 200)
+ end
+
+ test "calculates conversion_rate for goals with glob pattern with imported data", %{
+ conn: conn,
+ site: site
+ } do
+ site_import =
+ insert(:site_import,
+ start_date: ~D[2005-01-01],
+ end_date: Timex.today(),
+ source: :universal_analytics
+ )
+
+ populate_stats(site, site_import.id, [
+ build(:pageview, pathname: "/"),
+ build(:pageview, pathname: "/another"),
+ build(:pageview, pathname: "/blog/post-1"),
+ build(:pageview, pathname: "/blog/post-2"),
+ build(:imported_pages, page: "/blog/post-1"),
+ build(:imported_visitors)
+ ])
+
+ insert(:goal, %{site: site, page_path: "/blog**"})
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/conversions?period=day"
+ )
+
+ assert json_response(conn, 200) == [
+ %{
+ "name" => "Visit /blog**",
+ "visitors" => 2,
+ "events" => 2,
+ "conversion_rate" => 50
+ }
+ ]
+ end
+ end
end
diff --git a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs
index d53658806c..9c1898a02c 100644
--- a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs
@@ -1132,4 +1132,64 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
}
end
end
+
+ describe "with imported data" do
+ setup [:create_user, :log_in, :create_new_site]
+
+ for goal_name <- ["Outbound Link: Click", "File Download"] do
+ test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do
+ insert(:goal, event_name: unquote(goal_name), site: site)
+ site_import = insert(:site_import, site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:event,
+ name: unquote(goal_name),
+ "meta.key": ["url"],
+ "meta.value": ["https://one.com"]
+ ),
+ build(:imported_custom_events,
+ name: unquote(goal_name),
+ visitors: 2,
+ events: 5,
+ link_url: "https://one.com"
+ ),
+ build(:imported_custom_events,
+ name: unquote(goal_name),
+ visitors: 5,
+ events: 10,
+ link_url: "https://two.com"
+ ),
+ build(:imported_custom_events,
+ name: "view_search_results",
+ visitors: 100,
+ events: 200
+ ),
+ build(:imported_visitors, visitors: 9)
+ ])
+
+ filters = Jason.encode!(%{goal: unquote(goal_name)})
+
+ conn =
+ get(
+ conn,
+ "/api/stats/#{site.domain}/custom-prop-values/url?period=day&with_imported=true&filters=#{filters}"
+ )
+
+ assert json_response(conn, 200) == [
+ %{
+ "visitors" => 5,
+ "name" => "https://two.com",
+ "events" => 10,
+ "conversion_rate" => 50.0
+ },
+ %{
+ "visitors" => 3,
+ "name" => "https://one.com",
+ "events" => 6,
+ "conversion_rate" => 30.0
+ }
+ ]
+ end
+ end
+ end
end
diff --git a/test/support/factory.ex b/test/support/factory.ex
index 6ab32a7501..d2856ed21d 100644
--- a/test/support/factory.ex
+++ b/test/support/factory.ex
@@ -237,6 +237,17 @@ defmodule Plausible.Factory do
}
end
+ def imported_custom_events_factory do
+ %{
+ table: "imported_custom_events",
+ date: Timex.today(),
+ name: "",
+ link_url: "",
+ visitors: 1,
+ events: 1
+ }
+ end
+
def imported_locations_factory do
%{
table: "imported_locations",