Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Implement basics of GA4 import (#3851)

Adrian Gruntkowski4d7d88cfeccb
diff --git a/config/config.exs b/config/config.exs
index b3a1f05e88..f7f2f32d80 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -45,7 +45,7 @@ config :ref_inspector,
config :plausible,
paddle_api: Plausible.Billing.PaddleApi,
- google_api: Plausible.Google.Api
+ google_api: Plausible.Google.API
config :plausible,
# 30 minutes
diff --git a/config/test.exs b/config/test.exs
index 31182b2846..d1d9edfac0 100644
--- a/config/test.exs
+++ b/config/test.exs
@@ -16,7 +16,7 @@ config :plausible, Plausible.Mailer, adapter: Bamboo.TestAdapter
config :plausible,
paddle_api: Plausible.PaddleApi.Mock,
- google_api: Plausible.Google.Api.Mock
+ google_api: Plausible.Google.API.Mock
config :bamboo, :refute_timeout, 10
diff --git a/fixture/ga4_list_properties.json b/fixture/ga4_list_properties.json
new file mode 100644
index 0000000000..f8aac0b1d6
--- /dev/null
+++ b/fixture/ga4_list_properties.json
@@ -0,0 +1,41 @@
+{
+ "accountSummaries": [
+ {
+ "account": "accounts/28425178",
+ "displayName": "account.one",
+ "name": "accountSummaries/28425178",
+ "propertySummaries": [
+ {
+ "displayName": "account.one - GA4",
+ "parent": "accounts/28425178",
+ "property": "properties/428685906",
+ "propertyType": "PROPERTY_TYPE_ORDINARY"
+ }
+ ]
+ },
+ {
+ "account": "accounts/45336102",
+ "displayName": "account.two",
+ "name": "accountSummaries/45336102"
+ },
+ {
+ "account": "accounts/54516992",
+ "displayName": "Demo Account",
+ "name": "accountSummaries/54516992",
+ "propertySummaries": [
+ {
+ "displayName": "GA4 - Flood-It!",
+ "parent": "accounts/54516992",
+ "property": "properties/153293282",
+ "propertyType": "PROPERTY_TYPE_ORDINARY"
+ },
+ {
+ "displayName": "GA4 - Google Merch Shop",
+ "parent": "accounts/54516992",
+ "property": "properties/213025502",
+ "propertyType": "PROPERTY_TYPE_ORDINARY"
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fixture/ga4_report_imported_browsers.json b/fixture/ga4_report_imported_browsers.json
new file mode 100644
index 0000000000..214ac2b268
--- /dev/null
+++ b/fixture/ga4_report_imported_browsers.json
@@ -0,0 +1,161 @@
+{
+ "kind": "analyticsData#batchRunReports",
+ "reports": [
+ {
+ "dimensionHeaders": [
+ {
+ "name": "date"
+ },
+ {
+ "name": "browser"
+ }
+ ],
+ "kind": "analyticsData#runReport",
+ "metadata": {
+ "currencyCode": "USD",
+ "timeZone": "Europe/Warsaw"
+ },
+ "metricHeaders": [
+ {
+ "name": "totalUsers",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "sessions",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "bounces",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "userEngagementDuration",
+ "type": "TYPE_SECONDS"
+ }
+ ],
+ "rowCount": 5,
+ "rows": [
+ {
+ "dimensionValues": [
+ {
+ "value": "20240226"
+ },
+ {
+ "value": "Safari"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240224"
+ },
+ {
+ "value": "Chrome"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "2"
+ },
+ {
+ "value": "2"
+ },
+ {
+ "value": "2"
+ },
+ {
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240222"
+ },
+ {
+ "value": "Chrome"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "0"
+ },
+ {
+ "value": "4"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240222"
+ },
+ {
+ "value": "Firefox"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "2"
+ },
+ {
+ "value": "0"
+ },
+ {
+ "value": "21"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240222"
+ },
+ {
+ "value": "Safari"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "0"
+ },
+ {
+ "value": "4"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fixture/ga4_report_imported_devices.json b/fixture/ga4_report_imported_devices.json
new file mode 100644
index 0000000000..e8bcc01047
--- /dev/null
+++ b/fixture/ga4_report_imported_devices.json
@@ -0,0 +1,137 @@
+{
+ "kind": "analyticsData#batchRunReports",
+ "reports": [
+ {
+ "dimensionHeaders": [
+ {
+ "name": "date"
+ },
+ {
+ "name": "deviceCategory"
+ }
+ ],
+ "kind": "analyticsData#runReport",
+ "metadata": {
+ "currencyCode": "USD",
+ "timeZone": "Europe/Warsaw"
+ },
+ "metricHeaders": [
+ {
+ "name": "totalUsers",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "sessions",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "bounces",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "userEngagementDuration",
+ "type": "TYPE_SECONDS"
+ }
+ ],
+ "rowCount": 4,
+ "rows": [
+ {
+ "dimensionValues": [
+ {
+ "value": "20240226"
+ },
+ {
+ "value": "mobile"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240224"
+ },
+ {
+ "value": "desktop"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "2"
+ },
+ {
+ "value": "2"
+ },
+ {
+ "value": "2"
+ },
+ {
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240222"
+ },
+ {
+ "value": "desktop"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "2"
+ },
+ {
+ "value": "3"
+ },
+ {
+ "value": "0"
+ },
+ {
+ "value": "25"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240222"
+ },
+ {
+ "value": "mobile"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "0"
+ },
+ {
+ "value": "4"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fixture/ga4_report_imported_entry_pages.json b/fixture/ga4_report_imported_entry_pages.json
new file mode 100644
index 0000000000..77370bb30f
--- /dev/null
+++ b/fixture/ga4_report_imported_entry_pages.json
@@ -0,0 +1,137 @@
+{
+ "kind": "analyticsData#batchRunReports",
+ "reports": [
+ {
+ "dimensionHeaders": [
+ {
+ "name": "date"
+ },
+ {
+ "name": "landingPage"
+ }
+ ],
+ "kind": "analyticsData#runReport",
+ "metadata": {
+ "currencyCode": "USD",
+ "timeZone": "Europe/Warsaw"
+ },
+ "metricHeaders": [
+ {
+ "name": "totalUsers",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "sessions",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "userEngagementDuration",
+ "type": "TYPE_SECONDS"
+ },
+ {
+ "name": "bounces",
+ "type": "TYPE_INTEGER"
+ }
+ ],
+ "rowCount": 4,
+ "rows": [
+ {
+ "dimensionValues": [
+ {
+ "value": "20240226"
+ },
+ {
+ "value": "/blog/firstpost"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "0"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240224"
+ },
+ {
+ "value": "/"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "2"
+ },
+ {
+ "value": "2"
+ },
+ {
+ "value": "0"
+ },
+ {
+ "value": "2"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240222"
+ },
+ {
+ "value": "/"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "2"
+ },
+ {
+ "value": "3"
+ },
+ {
+ "value": "25"
+ },
+ {
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240222"
+ },
+ {
+ "value": "/blog/unicode-in-elixir"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "4"
+ },
+ {
+ "value": "0"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fixture/ga4_report_imported_locations.json b/fixture/ga4_report_imported_locations.json
new file mode 100644
index 0000000000..b731a21025
--- /dev/null
+++ b/fixture/ga4_report_imported_locations.json
@@ -0,0 +1,167 @@
+{
+ "kind": "analyticsData#batchRunReports",
+ "reports": [
+ {
+ "dimensionHeaders": [
+ {
+ "name": "date"
+ },
+ {
+ "name": "countryId"
+ },
+ {
+ "name": "region"
+ },
+ {
+ "name": "city"
+ }
+ ],
+ "kind": "analyticsData#runReport",
+ "metadata": {
+ "currencyCode": "USD",
+ "timeZone": "Europe/Warsaw"
+ },
+ "metricHeaders": [
+ {
+ "name": "totalUsers",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "sessions",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "bounces",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "userEngagementDuration",
+ "type": "TYPE_SECONDS"
+ }
+ ],
+ "rowCount": 4,
+ "rows": [
+ {
+ "dimensionValues": [
+ {
+ "value": "20240226"
+ },
+ {
+ "value": "PL"
+ },
+ {
+ "value": "Masovian Voivodeship"
+ },
+ {
+ "value": "Warsaw"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240224"
+ },
+ {
+ "value": "US"
+ },
+ {
+ "value": "California"
+ },
+ {
+ "value": "(not set)"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "2"
+ },
+ {
+ "value": "2"
+ },
+ {
+ "value": "2"
+ },
+ {
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240222"
+ },
+ {
+ "value": "PL"
+ },
+ {
+ "value": "Pomeranian Voivodeship"
+ },
+ {
+ "value": "Gdansk"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "2"
+ },
+ {
+ "value": "3"
+ },
+ {
+ "value": "0"
+ },
+ {
+ "value": "25"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240222"
+ },
+ {
+ "value": "ES"
+ },
+ {
+ "value": "Catalonia"
+ },
+ {
+ "value": "Barcelona"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "0"
+ },
+ {
+ "value": "4"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fixture/ga4_report_imported_operating_systems.json b/fixture/ga4_report_imported_operating_systems.json
new file mode 100644
index 0000000000..158fe1efa9
--- /dev/null
+++ b/fixture/ga4_report_imported_operating_systems.json
@@ -0,0 +1,137 @@
+{
+ "kind": "analyticsData#batchRunReports",
+ "reports": [
+ {
+ "dimensionHeaders": [
+ {
+ "name": "date"
+ },
+ {
+ "name": "operatingSystem"
+ }
+ ],
+ "kind": "analyticsData#runReport",
+ "metadata": {
+ "currencyCode": "USD",
+ "timeZone": "Europe/Warsaw"
+ },
+ "metricHeaders": [
+ {
+ "name": "totalUsers",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "sessions",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "bounces",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "userEngagementDuration",
+ "type": "TYPE_SECONDS"
+ }
+ ],
+ "rowCount": 4,
+ "rows": [
+ {
+ "dimensionValues": [
+ {
+ "value": "20240226"
+ },
+ {
+ "value": "iOS"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240224"
+ },
+ {
+ "value": "Windows"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "2"
+ },
+ {
+ "value": "2"
+ },
+ {
+ "value": "2"
+ },
+ {
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240222"
+ },
+ {
+ "value": "Macintosh"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "2"
+ },
+ {
+ "value": "3"
+ },
+ {
+ "value": "0"
+ },
+ {
+ "value": "25"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240222"
+ },
+ {
+ "value": "iOS"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "0"
+ },
+ {
+ "value": "4"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fixture/ga4_report_imported_pages.json b/fixture/ga4_report_imported_pages.json
new file mode 100644
index 0000000000..f7be04f265
--- /dev/null
+++ b/fixture/ga4_report_imported_pages.json
@@ -0,0 +1,232 @@
+{
+ "kind": "analyticsData#batchRunReports",
+ "reports": [
+ {
+ "dimensionHeaders": [
+ {
+ "name": "date"
+ },
+ {
+ "name": "hostName"
+ },
+ {
+ "name": "pagePath"
+ }
+ ],
+ "kind": "analyticsData#runReport",
+ "metadata": {
+ "currencyCode": "USD",
+ "timeZone": "Europe/Warsaw"
+ },
+ "metricHeaders": [
+ {
+ "name": "totalUsers",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "screenPageViews",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "userEngagementDuration",
+ "type": "TYPE_SECONDS"
+ }
+ ],
+ "rowCount": 8,
+ "rows": [
+ {
+ "dimensionValues": [
+ {
+ "value": "20240226"
+ },
+ {
+ "value": "drawer.todo.computer"
+ },
+ {
+ "value": "/blog/firstpost/"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240224"
+ },
+ {
+ "value": "drawer-4l3.pages.dev"
+ },
+ {
+ "value": "/"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240224"
+ },
+ {
+ "value": "drawer.todo.computer"
+ },
+ {
+ "value": "/"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240222"
+ },
+ {
+ "value": "drawer.todo.computer"
+ },
+ {
+ "value": "/"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "2"
+ },
+ {
+ "value": "3"
+ },
+ {
+ "value": "7"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240222"
+ },
+ {
+ "value": "drawer.todo.computer"
+ },
+ {
+ "value": "/blog/unicode-in-elixir/"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "2"
+ },
+ {
+ "value": "7"
+ },
+ {
+ "value": "21"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240222"
+ },
+ {
+ "value": "drawer.todo.computer"
+ },
+ {
+ "value": "/about/"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240222"
+ },
+ {
+ "value": "drawer.todo.computer"
+ },
+ {
+ "value": "/blog/"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240222"
+ },
+ {
+ "value": "drawer.todo.computer"
+ },
+ {
+ "value": "/blog/firstpost/"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "0"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fixture/ga4_report_imported_sources.json b/fixture/ga4_report_imported_sources.json
new file mode 100644
index 0000000000..4c5edb5179
--- /dev/null
+++ b/fixture/ga4_report_imported_sources.json
@@ -0,0 +1,161 @@
+{
+ "kind": "analyticsData#batchRunReports",
+ "reports": [
+ {
+ "dimensionHeaders": [
+ {
+ "name": "date"
+ },
+ {
+ "name": "sessionSource"
+ },
+ {
+ "name": "sessionMedium"
+ },
+ {
+ "name": "sessionCampaignName"
+ },
+ {
+ "name": "sessionManualAdContent"
+ },
+ {
+ "name": "sessionGoogleAdsKeyword"
+ }
+ ],
+ "kind": "analyticsData#runReport",
+ "metadata": {
+ "currencyCode": "USD",
+ "timeZone": "Europe/Warsaw"
+ },
+ "metricHeaders": [
+ {
+ "name": "totalUsers",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "sessions",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "bounces",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "userEngagementDuration",
+ "type": "TYPE_SECONDS"
+ }
+ ],
+ "rowCount": 3,
+ "rows": [
+ {
+ "dimensionValues": [
+ {
+ "value": "20240226"
+ },
+ {
+ "value": "(direct)"
+ },
+ {
+ "value": "(none)"
+ },
+ {
+ "value": "(direct)"
+ },
+ {
+ "value": "(not set)"
+ },
+ {
+ "value": "(not set)"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240224"
+ },
+ {
+ "value": "(direct)"
+ },
+ {
+ "value": "(none)"
+ },
+ {
+ "value": "(direct)"
+ },
+ {
+ "value": "(not set)"
+ },
+ {
+ "value": "(not set)"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "2"
+ },
+ {
+ "value": "2"
+ },
+ {
+ "value": "2"
+ },
+ {
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240222"
+ },
+ {
+ "value": "(direct)"
+ },
+ {
+ "value": "(none)"
+ },
+ {
+ "value": "(direct)"
+ },
+ {
+ "value": "(not set)"
+ },
+ {
+ "value": "(not set)"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "3"
+ },
+ {
+ "value": "4"
+ },
+ {
+ "value": "0"
+ },
+ {
+ "value": "29"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fixture/ga4_report_imported_visitors.json b/fixture/ga4_report_imported_visitors.json
new file mode 100644
index 0000000000..78d29b5e70
--- /dev/null
+++ b/fixture/ga4_report_imported_visitors.json
@@ -0,0 +1,114 @@
+{
+ "kind": "analyticsData#batchRunReports",
+ "reports": [
+ {
+ "dimensionHeaders": [
+ {
+ "name": "date"
+ }
+ ],
+ "kind": "analyticsData#runReport",
+ "metadata": {
+ "currencyCode": "USD",
+ "timeZone": "Europe/Warsaw"
+ },
+ "metricHeaders": [
+ {
+ "name": "totalUsers",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "screenPageViews",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "bounces",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "sessions",
+ "type": "TYPE_INTEGER"
+ },
+ {
+ "name": "userEngagementDuration",
+ "type": "TYPE_SECONDS"
+ }
+ ],
+ "rowCount": 3,
+ "rows": [
+ {
+ "dimensionValues": [
+ {
+ "value": "20240226"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "1"
+ },
+ {
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240224"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "2"
+ },
+ {
+ "value": "2"
+ },
+ {
+ "value": "2"
+ },
+ {
+ "value": "2"
+ },
+ {
+ "value": "0"
+ }
+ ]
+ },
+ {
+ "dimensionValues": [
+ {
+ "value": "20240222"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "3"
+ },
+ {
+ "value": "13"
+ },
+ {
+ "value": "0"
+ },
+ {
+ "value": "4"
+ },
+ {
+ "value": "29"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/fixture/ga4_start_date.json b/fixture/ga4_start_date.json
new file mode 100644
index 0000000000..6c1d410d23
--- /dev/null
+++ b/fixture/ga4_start_date.json
@@ -0,0 +1,38 @@
+{
+ "kind": "analyticsData#batchRunReports",
+ "reports": [
+ {
+ "dimensionHeaders": [
+ {
+ "name": "date"
+ }
+ ],
+ "kind": "analyticsData#runReport",
+ "metadata": {
+ "currencyCode": "USD",
+ "timeZone": "Europe/Warsaw"
+ },
+ "metricHeaders": [
+ {
+ "name": "screenPageViews",
+ "type": "TYPE_INTEGER"
+ }
+ ],
+ "rowCount": 3,
+ "rows": [
+ {
+ "dimensionValues": [
+ {
+ "value": "20240222"
+ }
+ ],
+ "metricValues": [
+ {
+ "value": "13"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex
index 085513886d..6b1e44b4ea 100644
--- a/lib/plausible/google/api.ex
+++ b/lib/plausible/google/api.ex
@@ -1,9 +1,13 @@
-defmodule Plausible.Google.Api do
- alias Plausible.Google.{ReportRequest, HTTP}
+defmodule Plausible.Google.API do
+ @moduledoc """
+ API to Google services.
+ """
+
use Timex
- require Logger
- @type google_analytics_view() :: {view_name :: String.t(), view_id :: String.t()}
+ alias Plausible.Google.HTTP
+
+ require Logger
@search_console_scope URI.encode_www_form(
"email https://www.googleapis.com/auth/webmasters.readonly"
@@ -17,9 +21,16 @@ defmodule Plausible.Google.Api do
Jason.encode!([site_id, redirect_to])
end
- def import_authorize_url(site_id, redirect_to, legacy \\ true) do
+ def import_authorize_url(site_id, redirect_to, opts \\ []) do
+ legacy = Keyword.get(opts, :legacy, true)
+ ga4 = Keyword.get(opts, :ga4, false)
+
"https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@import_scope}&state=" <>
- Jason.encode!([site_id, redirect_to, legacy])
+ Jason.encode!([site_id, redirect_to, legacy, ga4])
+ end
+
+ def fetch_access_token!(code) do
+ HTTP.fetch_access_token!(code)
end
def fetch_verified_properties(auth) do
@@ -53,138 +64,7 @@ defmodule Plausible.Google.Api do
end
end
- @spec list_views(access_token :: String.t()) ::
- {:ok, %{(hostname :: String.t()) => [google_analytics_view()]}} | {:error, term()}
- @doc """
- Lists Google Analytics views grouped by hostname.
- """
- def list_views(access_token) do
- case HTTP.list_views_for_user(access_token) do
- {:ok, %{"items" => views}} ->
- views = Enum.group_by(views, &view_hostname/1, &view_names/1)
- {:ok, views}
-
- error ->
- error
- end
- end
-
- defp view_hostname(view) do
- case view do
- %{"websiteUrl" => url} when is_binary(url) -> url |> URI.parse() |> Map.get(:host)
- _any -> "Others"
- end
- end
-
- defp view_names(%{"name" => name, "id" => id}) do
- {"#{id} - #{name}", id}
- end
-
- @spec get_view(access_token :: String.t(), lookup_id :: String.t()) ::
- {:ok, google_analytics_view()} | {:ok, nil} | {:error, term()}
- @doc """
- Returns a single Google Analytics view if the user has access to it.
- """
- def get_view(access_token, lookup_id) do
- case list_views(access_token) do
- {:ok, views} ->
- view =
- views
- |> Map.values()
- |> List.flatten()
- |> Enum.find(fn {_name, id} -> id == lookup_id end)
-
- {:ok, view}
-
- {:error, cause} ->
- {:error, cause}
- end
- end
-
- @type import_auth :: {
- access_token :: String.t(),
- refresh_token :: String.t(),
- expires_at :: String.t()
- }
-
- @per_page 7_500
- @backoff_factor :timer.seconds(10)
- @max_attempts 5
- @spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) ::
- :ok | {:error, term()}
- @doc """
- Imports stats from a Google Analytics UA view to a Plausible site.
-
- This function fetches Google Analytics reports in batches of #{@per_page} per
- request. The batches are then passed to persist callback.
-
- Requests to Google Analytics can fail, and are retried at most
- #{@max_attempts} times with an exponential backoff. Returns `:ok` when
- importing has finished or `{:error, term()}` when a request to GA failed too
- many times.
-
- Useful links:
-
- - [Feature documentation](https://plausible.io/docs/google-analytics-import)
- - [GA API reference](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest)
- - [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer)
-
- """
- def import_analytics(date_range, view_id, auth, persist_fn) do
- with {:ok, access_token} <- maybe_refresh_token(auth) do
- do_import_analytics(date_range, view_id, access_token, persist_fn)
- end
- end
-
- defp do_import_analytics(date_range, view_id, access_token, persist_fn) do
- Enum.reduce_while(ReportRequest.full_report(), :ok, fn report_request, :ok ->
- report_request = %ReportRequest{
- report_request
- | date_range: date_range,
- view_id: view_id,
- access_token: access_token,
- page_token: nil,
- page_size: @per_page
- }
-
- case fetch_and_persist(report_request, persist_fn: persist_fn) do
- :ok -> {:cont, :ok}
- {:error, _} = error -> {:halt, error}
- end
- end)
- end
-
- @spec fetch_and_persist(ReportRequest.t(), Keyword.t()) ::
- :ok | {:error, term()}
- def fetch_and_persist(%ReportRequest{} = report_request, opts \\ []) do
- persist_fn = Keyword.fetch!(opts, :persist_fn)
- attempt = Keyword.get(opts, :attempt, 1)
- sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)
-
- case HTTP.get_report(report_request) do
- {:ok, {rows, next_page_token}} ->
- :ok = persist_fn.(report_request.dataset, rows)
-
- if next_page_token do
- fetch_and_persist(
- %ReportRequest{report_request | page_token: next_page_token},
- opts
- )
- else
- :ok
- end
-
- {:error, cause} ->
- if attempt >= @max_attempts do
- {:error, cause}
- else
- Process.sleep(attempt * sleep_time)
- fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))
- end
- end
- end
-
- defp maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do
+ def maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do
with true <- needs_to_refresh_token?(auth.expires),
{:ok, {new_access_token, expires_at}} <- do_refresh_token(auth.refresh_token),
changeset <-
@@ -200,7 +80,7 @@ defmodule Plausible.Google.Api do
end
end
- defp maybe_refresh_token({access_token, refresh_token, expires_at}) do
+ def maybe_refresh_token({access_token, refresh_token, expires_at}) do
with true <- needs_to_refresh_token?(expires_at),
{:ok, {new_access_token, _expires_at}} <- do_refresh_token(refresh_token) do
{:ok, new_access_token}
diff --git a/lib/plausible/google/ga4/api.ex b/lib/plausible/google/ga4/api.ex
new file mode 100644
index 0000000000..f9062d00e3
--- /dev/null
+++ b/lib/plausible/google/ga4/api.ex
@@ -0,0 +1,142 @@
+defmodule Plausible.Google.GA4.API do
+ @moduledoc """
+ API for Google Analytics 4.
+ """
+
+ alias Plausible.Google
+ alias Plausible.Google.GA4
+
+ require Logger
+
+ @type import_auth :: {
+ access_token :: String.t(),
+ refresh_token :: String.t(),
+ expires_at :: String.t()
+ }
+
+ @per_page 50_000
+ @backoff_factor :timer.seconds(10)
+ @max_attempts 5
+
+ def list_properties(access_token) do
+ case GA4.HTTP.list_accounts_for_user(access_token) do
+ {:ok, %{"accountSummaries" => accounts}} ->
+ accounts =
+ accounts
+ |> Enum.filter(& &1["propertySummaries"])
+ |> Enum.map(fn account ->
+ {"#{account["displayName"]} (#{account["account"]})",
+ Enum.map(account["propertySummaries"], fn property ->
+ {"#{property["displayName"]} (#{property["property"]})", property["property"]}
+ end)}
+ end)
+
+ {:ok, accounts}
+
+ error ->
+ error
+ end
+ end
+
+ def get_property(access_token, lookup_property) do
+ case list_properties(access_token) do
+ {:ok, properties} ->
+ property =
+ properties
+ |> Enum.map(&elem(&1, 1))
+ |> List.flatten()
+ |> Enum.find(fn {_name, property} -> property == lookup_property end)
+
+ {:ok, property}
+
+ {:error, cause} ->
+ {:error, cause}
+ end
+ end
+
+ def get_analytics_start_date(access_token, property) do
+ GA4.HTTP.get_analytics_start_date(access_token, property)
+ end
+
+ def import_analytics(date_range, property, auth, persist_fn) do
+ Logger.debug(
+ "[#{inspect(__MODULE__)}:#{property}] Starting import from #{date_range.first} to #{date_range.last}"
+ )
+
+ with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do
+ do_import_analytics(date_range, property, access_token, persist_fn)
+ end
+ end
+
+ defp do_import_analytics(date_range, property, access_token, persist_fn) do
+ Enum.reduce_while(GA4.ReportRequest.full_report(), :ok, fn report_request, :ok ->
+ Logger.debug(
+ "[#{inspect(__MODULE__)}:#{property}] Starting to import #{report_request.dataset}"
+ )
+
+ report_request = prepare_request(report_request, date_range, property, access_token)
+
+ case fetch_and_persist(report_request, persist_fn: persist_fn) do
+ :ok -> {:cont, :ok}
+ {:error, _} = error -> {:halt, error}
+ end
+ end)
+ end
+
+ @spec fetch_and_persist(GA4.ReportRequest.t(), Keyword.t()) ::
+ :ok | {:error, term()}
+ def fetch_and_persist(%GA4.ReportRequest{} = report_request, opts \\ []) do
+ persist_fn = Keyword.fetch!(opts, :persist_fn)
+ attempt = Keyword.get(opts, :attempt, 1)
+ sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)
+
+ case GA4.HTTP.get_report(report_request) do
+ {:ok, {rows, row_count}} ->
+ Logger.debug(
+ "[#{inspect(__MODULE__)}:#{report_request.property}] Fetched #{length(rows)} rows of total #{row_count} with offset #{report_request.offset} for #{report_request.dataset}"
+ )
+
+ :ok = persist_fn.(report_request.dataset, rows)
+
+ Logger.debug(
+ "[#{inspect(__MODULE__)}:#{report_request.property}] Persisted #{length(rows)} for #{report_request.dataset}"
+ )
+
+ if report_request.offset + @per_page < row_count do
+ fetch_and_persist(
+ %GA4.ReportRequest{report_request | offset: report_request.offset + @per_page},
+ opts
+ )
+ else
+ :ok
+ end
+
+ {:error, cause} ->
+ if attempt >= @max_attempts do
+ Logger.debug(
+ "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Terminating."
+ )
+
+ {:error, cause}
+ else
+ Logger.debug(
+ "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Will retry."
+ )
+
+ Process.sleep(attempt * sleep_time)
+ fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))
+ end
+ end
+ end
+
+ defp prepare_request(report_request, date_range, property, access_token) do
+ %GA4.ReportRequest{
+ report_request
+ | date_range: date_range,
+ property: property,
+ access_token: access_token,
+ offset: 0,
+ limit: @per_page
+ }
+ end
+end
diff --git a/lib/plausible/google/ga4/http.ex b/lib/plausible/google/ga4/http.ex
new file mode 100644
index 0000000000..da9ace19b2
--- /dev/null
+++ b/lib/plausible/google/ga4/http.ex
@@ -0,0 +1,199 @@
+defmodule Plausible.Google.GA4.HTTP do
+ @moduledoc """
+ HTTP client implementation for Google Analytics 4 API.
+ """
+
+ alias Plausible.HTTPClient
+
+ require Logger
+
+ @spec get_report(Plausible.Google.GA4.ReportRequest.t()) ::
+ {:ok, {[map()], non_neg_integer()}} | {:error, any()}
+ def get_report(%Plausible.Google.GA4.ReportRequest{} = report_request) do
+ params = %{
+ requests: [
+ %{
+ property: report_request.property,
+ dateRanges: [
+ %{
+ startDate: report_request.date_range.first,
+ endDate: report_request.date_range.last
+ }
+ ],
+ dimensions: Enum.map(report_request.dimensions, &%{name: &1}),
+ metrics: Enum.map(report_request.metrics, &build_metric/1),
+ orderBys: [
+ %{
+ dimension: %{
+ dimensionName: "date",
+ orderType: "ALPHANUMERIC"
+ },
+ desc: true
+ }
+ ],
+ limit: report_request.limit,
+ offset: report_request.offset
+ }
+ ]
+ }
+
+ url =
+ "#{reporting_api_url()}/v1beta/#{report_request.property}:batchRunReports"
+
+ response =
+ HTTPClient.impl().post(
+ url,
+ [{"Authorization", "Bearer #{report_request.access_token}"}],
+ params,
+ receive_timeout: 60_000
+ )
+
+ with {:ok, %{body: body}} <- response,
+ # File.write!("fixture/ga4_report_#{report_request.dataset}.json", Jason.encode!(body)),
+ {:ok, report} <- parse_report_from_response(body),
+ row_count <- Map.get(report, "rowCount"),
+ {:ok, report} <- convert_to_maps(report) do
+ {:ok, {report, row_count}}
+ else
+ {:error, %{reason: %{status: status, body: body}}} ->
+ Logger.debug(
+ "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} with code #{status}: #{inspect(body)}"
+ )
+
+ Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
+ {:error, :request_failed}
+
+ {:error, reason} ->
+ Logger.debug(
+ "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}: #{inspect(reason)}"
+ )
+
+ {:error, :request_failed}
+ end
+ end
+
+ defp build_metric(expression) do
+ case String.split(expression, " = ") do
+ [name, expression] ->
+ %{
+ name: name,
+ expression: expression
+ }
+
+ [name] ->
+ %{name: name}
+ end
+ end
+
+ defp parse_report_from_response(%{"reports" => [report | _]}) do
+ {:ok, report}
+ end
+
+ defp parse_report_from_response(body) do
+ Sentry.Context.set_extra_context(%{google_analytics4_response: body})
+
+ Logger.error(
+ "Google Analytics 4: Failed to find report in response. Reason: #{inspect(body)}"
+ )
+
+ {:error, {:invalid_response, body}}
+ end
+
+ defp convert_to_maps(%{
+ "rows" => rows,
+ "dimensionHeaders" => dimension_headers,
+ "metricHeaders" => metric_headers
+ })
+ when is_list(rows) do
+ dimension_headers = Enum.map(dimension_headers, & &1["name"])
+ metric_headers = Enum.map(metric_headers, & &1["name"])
+
+ report =
+ Enum.map(rows, fn %{"dimensionValues" => dimensions, "metricValues" => metrics} ->
+ dimension_values = Enum.map(dimensions, & &1["value"])
+ metric_values = Enum.map(metrics, & &1["value"])
+ metrics = Enum.zip(metric_headers, metric_values)
+ dimensions = Enum.zip(dimension_headers, dimension_values)
+ %{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}
+ end)
+
+ {:ok, report}
+ end
+
+ defp convert_to_maps(response) do
+ Logger.error(
+ "Google Analytics 4: Failed to read report in response. Reason: #{inspect(response)}"
+ )
+
+ Sentry.Context.set_extra_context(%{google_analytics4_response: response})
+ {:error, {:invalid_response, response}}
+ end
+
+ def list_accounts_for_user(access_token) do
+ url = "#{admin_api_url()}/v1beta/accountSummaries"
+
+ headers = [{"Authorization", "Bearer #{access_token}"}]
+
+ case HTTPClient.impl().get(url, headers) do
+ {:ok, %Finch.Response{body: body, status: 200}} ->
+ {:ok, body}
+
+ {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
+ {:error, :authentication_failed}
+
+ {:error, %HTTPClient.Non200Error{} = error} ->
+ Sentry.capture_message("Error listing Google accounts for user", extra: %{error: error})
+ {:error, :unknown}
+ end
+ end
+
+ @earliest_valid_date "2015-08-14"
+ def get_analytics_start_date(access_token, property) do
+ params = %{
+ requests: [
+ %{
+ property: "#{property}",
+ dateRanges: [
+ %{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}
+ ],
+ dimensions: [%{name: "date"}],
+ metrics: [%{name: "screenPageViews"}],
+ orderBys: [
+ %{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: false}
+ ],
+ limit: 1
+ }
+ ]
+ }
+
+ url = "#{reporting_api_url()}/v1beta/#{property}:batchRunReports"
+ headers = [{"Authorization", "Bearer #{access_token}"}]
+
+ case HTTPClient.impl().post(url, headers, params) do
+ {:ok, %Finch.Response{body: body, status: 200}} ->
+ report = List.first(body["reports"])
+
+ date =
+ case report["rows"] do
+ [%{"dimensionValues" => [%{"value" => date_str}]}] ->
+ Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
+
+ _ ->
+ nil
+ end
+
+ {:ok, date}
+
+ {:error, %{reason: %Finch.Response{body: body}}} ->
+ Sentry.capture_message("Error fetching GA4 start date", extra: %{body: inspect(body)})
+ {:error, body}
+
+ {:error, %{reason: reason} = e} ->
+ Sentry.capture_message("Error fetching GA4 start date", extra: %{error: inspect(e)})
+ {:error, reason}
+ end
+ end
+
+ defp reporting_api_url, do: "https://analyticsdata.googleapis.com"
+ defp admin_api_url, do: "https://analyticsadmin.googleapis.com"
+end
diff --git a/lib/plausible/google/ga4/report_request.ex b/lib/plausible/google/ga4/report_request.ex
new file mode 100644
index 0000000000..b859f17893
--- /dev/null
+++ b/lib/plausible/google/ga4/report_request.ex
@@ -0,0 +1,122 @@
+defmodule Plausible.Google.GA4.ReportRequest do
+ @moduledoc """
+ Report request struct for Google Analytics 4 API
+ """
+
+ defstruct [
+ :dataset,
+ :dimensions,
+ :metrics,
+ :date_range,
+ :property,
+ :access_token,
+ :offset,
+ :limit
+ ]
+
+ @type t() :: %__MODULE__{
+ dataset: String.t(),
+ dimensions: [String.t()],
+ metrics: [String.t()],
+ date_range: Date.Range.t(),
+ property: term(),
+ access_token: String.t(),
+ offset: non_neg_integer(),
+ limit: non_neg_integer()
+ }
+
+ def full_report do
+ [
+ %__MODULE__{
+ dataset: "imported_visitors",
+ dimensions: ["date"],
+ metrics: [
+ "totalUsers",
+ "screenPageViews",
+ "bounces = sessions - engagedSessions",
+ "sessions",
+ "userEngagementDuration"
+ ]
+ },
+ %__MODULE__{
+ dataset: "imported_sources",
+ dimensions: [
+ "date",
+ "sessionSource",
+ "sessionMedium",
+ "sessionCampaignName",
+ "sessionManualAdContent",
+ "sessionGoogleAdsKeyword"
+ ],
+ metrics: [
+ "totalUsers",
+ "sessions",
+ "bounces = sessions - engagedSessions",
+ "userEngagementDuration"
+ ]
+ },
+ %__MODULE__{
+ dataset: "imported_pages",
+ dimensions: ["date", "hostName", "pagePath"],
+ # NOTE: no exits as GA4 DATA API does not provide that metric
+ metrics: ["totalUsers", "screenPageViews", "userEngagementDuration"]
+ },
+ %__MODULE__{
+ dataset: "imported_entry_pages",
+ dimensions: ["date", "landingPage"],
+ metrics: [
+ "totalUsers",
+ "sessions",
+ "userEngagementDuration",
+ "bounces = sessions - engagedSessions"
+ ]
+ },
+ # NOTE: Skipping for now as there's no dimension directly mapping to exit page path
+ # %__MODULE__{
+ # dataset: "imported_exit_pages",
+ # dimensions: ["date", "ga:exitPagePath"],
+ # metrics: ["totalUsers", "sessions"]
+ # },
+ %__MODULE__{
+ dataset: "imported_locations",
+ dimensions: ["date", "countryId", "region", "city"],
+ metrics: [
+ "totalUsers",
+ "sessions",
+ "bounces = sessions - engagedSessions",
+ "userEngagementDuration"
+ ]
+ },
+ %__MODULE__{
+ dataset: "imported_devices",
+ dimensions: ["date", "deviceCategory"],
+ metrics: [
+ "totalUsers",
+ "sessions",
+ "bounces = sessions - engagedSessions",
+ "userEngagementDuration"
+ ]
+ },
+ %__MODULE__{
+ dataset: "imported_browsers",
+ dimensions: ["date", "browser"],
+ metrics: [
+ "totalUsers",
+ "sessions",
+ "bounces = sessions - engagedSessions",
+ "userEngagementDuration"
+ ]
+ },
+ %__MODULE__{
+ dataset: "imported_operating_systems",
+ dimensions: ["date", "operatingSystem"],
+ metrics: [
+ "totalUsers",
+ "sessions",
+ "bounces = sessions - engagedSessions",
+ "userEngagementDuration"
+ ]
+ }
+ ]
+ end
+end
diff --git a/lib/plausible/google/http.ex b/lib/plausible/google/http.ex
index 93dad5910f..be55fbcf5e 100644
--- a/lib/plausible/google/http.ex
+++ b/lib/plausible/google/http.ex
@@ -2,97 +2,6 @@ defmodule Plausible.Google.HTTP do
require Logger
alias Plausible.HTTPClient
- @spec get_report(Plausible.Google.ReportRequest.t()) ::
- {:ok, {[map()], String.t() | nil}} | {:error, any()}
- def get_report(%Plausible.Google.ReportRequest{} = report_request) do
- params = %{
- reportRequests: [
- %{
- viewId: report_request.view_id,
- dateRanges: [
- %{
- startDate: report_request.date_range.first,
- endDate: report_request.date_range.last
- }
- ],
- dimensions: Enum.map(report_request.dimensions, &%{name: &1, histogramBuckets: []}),
- metrics: Enum.map(report_request.metrics, &%{expression: &1}),
- hideTotals: true,
- hideValueRanges: true,
- orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}],
- pageSize: report_request.page_size,
- pageToken: report_request.page_token
- }
- ]
- }
-
- response =
- HTTPClient.impl().post(
- "#{reporting_api_url()}/v4/reports:batchGet",
- [{"Authorization", "Bearer #{report_request.access_token}"}],
- params,
- receive_timeout: 60_000
- )
-
- with {:ok, %{body: body}} <- response,
- {:ok, report} <- parse_report_from_response(body),
- token <- Map.get(report, "nextPageToken"),
- {:ok, report} <- convert_to_maps(report) do
- {:ok, {report, token}}
- else
- {:error, %{reason: %{status: status, body: body}}} ->
- Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
- {:error, :request_failed}
-
- {:error, _reason} ->
- {:error, :request_failed}
- end
- end
-
- defp parse_report_from_response(body) do
- with %{"reports" => [report | _]} <- body do
- {:ok, report}
- else
- _ ->
- Sentry.Context.set_extra_context(%{google_analytics_response: body})
-
- Logger.error(
- "Google Analytics: Failed to find report in response. Reason: #{inspect(body)}"
- )
-
- {:error, {:invalid_response, body}}
- end
- end
-
- defp convert_to_maps(%{
- "data" => %{} = data,
- "columnHeader" => %{
- "dimensions" => dimension_headers,
- "metricHeader" => %{"metricHeaderEntries" => metric_headers}
- }
- }) do
- metric_headers = Enum.map(metric_headers, & &1["name"])
- rows = Map.get(data, "rows", [])
-
- report =
- Enum.map(rows, fn %{"dimensions" => dimensions, "metrics" => [%{"values" => metrics}]} ->
- metrics = Enum.zip(metric_headers, metrics)
- dimensions = Enum.zip(dimension_headers, dimensions)
- %{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}
- end)
-
- {:ok, report}
- end
-
- defp convert_to_maps(response) do
- Logger.error(
- "Google Analytics: Failed to read report in response. Reason: #{inspect(response)}"
- )
-
- Sentry.Context.set_extra_context(%{google_analytics_response: response})
- {:error, {:invalid_response, response}}
- end
-
def list_sites(access_token) do
url = "#{api_url()}/webmasters/v3/sites"
headers = [{"Content-Type", "application/json"}, {"Authorization", "Bearer #{access_token}"}]
@@ -113,7 +22,7 @@ defmodule Plausible.Google.HTTP do
end
end
- def fetch_access_token(code) do
+ def fetch_access_token!(code) do
url = "#{api_url()}/oauth2/v4/token"
headers = [{"Content-Type", "application/x-www-form-urlencoded"}]
@@ -130,24 +39,6 @@ defmodule Plausible.Google.HTTP do
response.body
end
- def list_views_for_user(access_token) do
- url = "#{api_url()}/analytics/v3/management/accounts/~all/webproperties/~all/profiles"
-
- headers = [{"Authorization", "Bearer #{access_token}"}]
-
- case HTTPClient.impl().get(url, headers) do
- {:ok, %Finch.Response{body: body, status: 200}} ->
- {:ok, body}
-
- {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
- {:error, :authentication_failed}
-
- {:error, %HTTPClient.Non200Error{} = error} ->
- Sentry.capture_message("Error listing GA views for user", extra: %{error: error})
- {:error, :unknown}
- end
- end
-
def list_stats(access_token, property, date_range, limit, page \\ nil) do
property = URI.encode_www_form(property)
@@ -215,57 +106,9 @@ defmodule Plausible.Google.HTTP do
end
end
- @earliest_valid_date "2005-01-01"
- def get_analytics_start_date(view_id, access_token) do
- params = %{
- reportRequests: [
- %{
- viewId: view_id,
- dateRanges: [
- %{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}
- ],
- dimensions: [%{name: "ga:date", histogramBuckets: []}],
- metrics: [%{expression: "ga:pageviews"}],
- hideTotals: true,
- hideValueRanges: true,
- orderBys: [%{fieldName: "ga:date", sortOrder: "ASCENDING"}],
- pageSize: 1
- }
- ]
- }
-
- url = "#{reporting_api_url()}/v4/reports:batchGet"
- headers = [{"Authorization", "Bearer #{access_token}"}]
-
- case HTTPClient.post(url, headers, params) do
- {:ok, %Finch.Response{body: body, status: 200}} ->
- report = List.first(body["reports"])
-
- date =
- case report["data"]["rows"] do
- [%{"dimensions" => [date_str]}] ->
- Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
-
- _ ->
- nil
- end
-
- {:ok, date}
-
- {:error, %{reason: %Finch.Response{body: body}}} ->
- Sentry.capture_message("Error fetching Google view ID", extra: %{body: inspect(body)})
- {:error, body}
-
- {:error, %{reason: reason} = e} ->
- Sentry.capture_message("Error fetching Google view ID", extra: %{error: inspect(e)})
- {:error, reason}
- end
- end
-
defp config, do: Application.get_env(:plausible, :google)
defp client_id, do: Keyword.fetch!(config(), :client_id)
defp client_secret, do: Keyword.fetch!(config(), :client_secret)
- defp reporting_api_url, do: Keyword.fetch!(config(), :reporting_api_url)
defp api_url, do: Keyword.fetch!(config(), :api_url)
defp redirect_uri, do: PlausibleWeb.Endpoint.url() <> "/auth/google/callback"
end
diff --git a/lib/plausible/google/ua/api.ex b/lib/plausible/google/ua/api.ex
new file mode 100644
index 0000000000..47d2cebebf
--- /dev/null
+++ b/lib/plausible/google/ua/api.ex
@@ -0,0 +1,146 @@
+defmodule Plausible.Google.UA.API do
+ @moduledoc """
+ API for Universal Analytics
+ """
+
+ alias Plausible.Google
+ alias Plausible.Google.UA
+
+ @type google_analytics_view() :: {view_name :: String.t(), view_id :: String.t()}
+
+ @type import_auth :: {
+ access_token :: String.t(),
+ refresh_token :: String.t(),
+ expires_at :: String.t()
+ }
+
+ @per_page 7_500
+ @backoff_factor :timer.seconds(10)
+ @max_attempts 5
+
+ @spec list_views(access_token :: String.t()) ::
+ {:ok, %{(hostname :: String.t()) => [google_analytics_view()]}} | {:error, term()}
+ @doc """
+ Lists Google Analytics views grouped by hostname.
+ """
+ def list_views(access_token) do
+ case UA.HTTP.list_views_for_user(access_token) do
+ {:ok, %{"items" => views}} ->
+ views = Enum.group_by(views, &view_hostname/1, &view_names/1)
+ {:ok, views}
+
+ error ->
+ error
+ end
+ end
+
+ @spec get_view(access_token :: String.t(), lookup_id :: String.t()) ::
+ {:ok, google_analytics_view()} | {:ok, nil} | {:error, term()}
+ @doc """
+ Returns a single Google Analytics view if the user has access to it.
+ """
+ def get_view(access_token, lookup_id) do
+ case list_views(access_token) do
+ {:ok, views} ->
+ view =
+ views
+ |> Map.values()
+ |> List.flatten()
+ |> Enum.find(fn {_name, id} -> id == lookup_id end)
+
+ {:ok, view}
+
+ {:error, cause} ->
+ {:error, cause}
+ end
+ end
+
+ def get_analytics_start_date(access_token, view_id) do
+ UA.HTTP.get_analytics_start_date(access_token, view_id)
+ end
+
+ @spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) ::
+ :ok | {:error, term()}
+ @doc """
+ Imports stats from a Google Analytics UA view to a Plausible site.
+
+ This function fetches Google Analytics reports in batches of #{@per_page} per
+ request. The batches are then passed to persist callback.
+
+ Requests to Google Analytics can fail, and are retried at most
+ #{@max_attempts} times with an exponential backoff. Returns `:ok` when
+ importing has finished or `{:error, term()}` when a request to GA failed too
+ many times.
+
+ Useful links:
+
+ - [Feature documentation](https://plausible.io/docs/google-analytics-import)
+ - [GA API reference](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest)
+ - [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer)
+
+ """
+ def import_analytics(date_range, view_id, auth, persist_fn) do
+ with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do
+ do_import_analytics(date_range, view_id, access_token, persist_fn)
+ end
+ end
+
+ defp do_import_analytics(date_range, view_id, access_token, persist_fn) do
+ Enum.reduce_while(UA.ReportRequest.full_report(), :ok, fn report_request, :ok ->
+ report_request = %UA.ReportRequest{
+ report_request
+ | date_range: date_range,
+ view_id: view_id,
+ access_token: access_token,
+ page_token: nil,
+ page_size: @per_page
+ }
+
+ case fetch_and_persist(report_request, persist_fn: persist_fn) do
+ :ok -> {:cont, :ok}
+ {:error, _} = error -> {:halt, error}
+ end
+ end)
+ end
+
+ @spec fetch_and_persist(UA.ReportRequest.t(), Keyword.t()) ::
+ :ok | {:error, term()}
+ def fetch_and_persist(%UA.ReportRequest{} = report_request, opts \\ []) do
+ persist_fn = Keyword.fetch!(opts, :persist_fn)
+ attempt = Keyword.get(opts, :attempt, 1)
+ sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)
+
+ case UA.HTTP.get_report(report_request) do
+ {:ok, {rows, next_page_token}} ->
+ :ok = persist_fn.(report_request.dataset, rows)
+
+ if next_page_token do
+ fetch_and_persist(
+ %UA.ReportRequest{report_request | page_token: next_page_token},
+ opts
+ )
+ else
+ :ok
+ end
+
+ {:error, cause} ->
+ if attempt >= @max_attempts do
+ {:error, cause}
+ else
+ Process.sleep(attempt * sleep_time)
+ fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))
+ end
+ end
+ end
+
+ defp view_hostname(view) do
+ case view do
+ %{"websiteUrl" => url} when is_binary(url) -> url |> URI.parse() |> Map.get(:host)
+ _any -> "Others"
+ end
+ end
+
+ defp view_names(%{"name" => name, "id" => id}) do
+ {"#{id} - #{name}", id}
+ end
+end
diff --git a/lib/plausible/google/ua/http.ex b/lib/plausible/google/ua/http.ex
new file mode 100644
index 0000000000..f00712f5ec
--- /dev/null
+++ b/lib/plausible/google/ua/http.ex
@@ -0,0 +1,167 @@
+defmodule Plausible.Google.UA.HTTP do
+ @moduledoc """
+ HTTP client implementation for Universal Analytics API.
+ """
+
+ require Logger
+ alias Plausible.HTTPClient
+
+ @spec get_report(Plausible.Google.UA.ReportRequest.t()) ::
+ {:ok, {[map()], String.t() | nil}} | {:error, any()}
+ def get_report(%Plausible.Google.UA.ReportRequest{} = report_request) do
+ params = %{
+ reportRequests: [
+ %{
+ viewId: report_request.view_id,
+ dateRanges: [
+ %{
+ startDate: report_request.date_range.first,
+ endDate: report_request.date_range.last
+ }
+ ],
+ dimensions: Enum.map(report_request.dimensions, &%{name: &1, histogramBuckets: []}),
+ metrics: Enum.map(report_request.metrics, &%{expression: &1}),
+ hideTotals: true,
+ hideValueRanges: true,
+ orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}],
+ pageSize: report_request.page_size,
+ pageToken: report_request.page_token
+ }
+ ]
+ }
+
+ response =
+ HTTPClient.impl().post(
+ "#{reporting_api_url()}/v4/reports:batchGet",
+ [{"Authorization", "Bearer #{report_request.access_token}"}],
+ params,
+ receive_timeout: 60_000
+ )
+
+ with {:ok, %{body: body}} <- response,
+ {:ok, report} <- parse_report_from_response(body),
+ token <- Map.get(report, "nextPageToken"),
+ {:ok, report} <- convert_to_maps(report) do
+ {:ok, {report, token}}
+ else
+ {:error, %{reason: %{status: status, body: body}}} ->
+ Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
+ {:error, :request_failed}
+
+ {:error, _reason} ->
+ {:error, :request_failed}
+ end
+ end
+
+ defp parse_report_from_response(%{"reports" => [report | _]}) do
+ {:ok, report}
+ end
+
+ defp parse_report_from_response(body) do
+ Sentry.Context.set_extra_context(%{universal_analytics_response: body})
+
+ Logger.error(
+ "Universal Analytics: Failed to find report in response. Reason: #{inspect(body)}"
+ )
+
+ {:error, {:invalid_response, body}}
+ end
+
+ defp convert_to_maps(%{
+ "data" => %{} = data,
+ "columnHeader" => %{
+ "dimensions" => dimension_headers,
+ "metricHeader" => %{"metricHeaderEntries" => metric_headers}
+ }
+ }) do
+ metric_headers = Enum.map(metric_headers, & &1["name"])
+ rows = Map.get(data, "rows", [])
+
+ report =
+ Enum.map(rows, fn %{"dimensions" => dimensions, "metrics" => [%{"values" => metrics}]} ->
+ metrics = Enum.zip(metric_headers, metrics)
+ dimensions = Enum.zip(dimension_headers, dimensions)
+ %{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}
+ end)
+
+ {:ok, report}
+ end
+
+ defp convert_to_maps(response) do
+ Logger.error(
+ "Universal Analytics: Failed to read report in response. Reason: #{inspect(response)}"
+ )
+
+ Sentry.Context.set_extra_context(%{universal_analytics_response: response})
+ {:error, {:invalid_response, response}}
+ end
+
+ def list_views_for_user(access_token) do
+ url = "#{api_url()}/analytics/v3/management/accounts/~all/webproperties/~all/profiles"
+
+ headers = [{"Authorization", "Bearer #{access_token}"}]
+
+ case HTTPClient.impl().get(url, headers) do
+ {:ok, %Finch.Response{body: body, status: 200}} ->
+ {:ok, body}
+
+ {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
+ {:error, :authentication_failed}
+
+ {:error, %HTTPClient.Non200Error{} = error} ->
+ Sentry.capture_message("Error listing GA views for user", extra: %{error: error})
+ {:error, :unknown}
+ end
+ end
+
+ @earliest_valid_date "2005-01-01"
+ def get_analytics_start_date(access_token, view_id) do
+ params = %{
+ reportRequests: [
+ %{
+ viewId: view_id,
+ dateRanges: [
+ %{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}
+ ],
+ dimensions: [%{name: "ga:date", histogramBuckets: []}],
+ metrics: [%{expression: "ga:pageviews"}],
+ hideTotals: true,
+ hideValueRanges: true,
+ orderBys: [%{fieldName: "ga:date", sortOrder: "ASCENDING"}],
+ pageSize: 1
+ }
+ ]
+ }
+
+ url = "#{reporting_api_url()}/v4/reports:batchGet"
+ headers = [{"Authorization", "Bearer #{access_token}"}]
+
+ case HTTPClient.post(url, headers, params) do
+ {:ok, %Finch.Response{body: body, status: 200}} ->
+ report = List.first(body["reports"])
+
+ date =
+ case report["data"]["rows"] do
+ [%{"dimensions" => [date_str]}] ->
+ Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
+
+ _ ->
+ nil
+ end
+
+ {:ok, date}
+
+ {:error, %{reason: %Finch.Response{body: body}}} ->
+ Sentry.capture_message("Error fetching UA start date", extra: %{body: inspect(body)})
+ {:error, body}
+
+ {:error, %{reason: reason} = e} ->
+ Sentry.capture_message("Error fetching UA start date", extra: %{error: inspect(e)})
+ {:error, reason}
+ end
+ end
+
+ defp config, do: Application.get_env(:plausible, :google)
+ defp reporting_api_url, do: Keyword.fetch!(config(), :reporting_api_url)
+ defp api_url, do: Keyword.fetch!(config(), :api_url)
+end
diff --git a/lib/plausible/google/report_request.ex b/lib/plausible/google/ua/report_request.ex
similarity index 95%
rename from lib/plausible/google/report_request.ex
rename to lib/plausible/google/ua/report_request.ex
index fcc8fe2c12..e55419dd7d 100644
--- a/lib/plausible/google/report_request.ex
+++ b/lib/plausible/google/ua/report_request.ex
@@ -1,4 +1,8 @@
-defmodule Plausible.Google.ReportRequest do
+defmodule Plausible.Google.UA.ReportRequest do
+ @moduledoc """
+ Report request struct for Universal Analytics API
+ """
+
defstruct [
:dataset,
:dimensions,
diff --git a/lib/plausible/imported/buffer.ex b/lib/plausible/imported/buffer.ex
index 554cfaf701..fccbb62784 100644
--- a/lib/plausible/imported/buffer.ex
+++ b/lib/plausible/imported/buffer.ex
@@ -8,12 +8,13 @@ defmodule Plausible.Imported.Buffer do
use GenServer
require Logger
- def start_link do
- GenServer.start_link(__MODULE__, nil)
+ def start_link(opts \\ []) do
+ GenServer.start_link(__MODULE__, opts)
end
- def init(_opts) do
- {:ok, %{buffers: %{}}}
+ def init(opts) do
+ flush_interval = Keyword.get(opts, :flush_interval_ms, 1000)
+ {:ok, %{flush_interval: flush_interval, buffers: %{}}}
end
@spec insert_many(pid(), term(), [map()]) :: :ok
@@ -68,14 +69,14 @@ defmodule Plausible.Imported.Buffer do
def handle_call(:flush_all_buffers, _from, state) do
Enum.each(state.buffers, fn {table_name, records} ->
- flush_buffer(records, table_name)
+ flush_buffer(records, table_name, state.flush_interval)
end)
{:reply, :ok, put_in(state.buffers, %{})}
end
def handle_continue({:flush, table_name}, state) do
- flush_buffer(state.buffers[table_name], table_name)
+ flush_buffer(state.buffers[table_name], table_name, state.flush_interval)
{:noreply, put_in(state.buffers[table_name], [])}
end
@@ -85,10 +86,10 @@ defmodule Plausible.Imported.Buffer do
|> Keyword.fetch!(:max_buffer_size)
end
- defp flush_buffer(records, table_name) do
+ defp flush_buffer(records, table_name, flush_interval) do
# Clickhouse does not recommend sending more than 1 INSERT operation per second, and this
# sleep call slows down the flushing
- Process.sleep(1000)
+ Process.sleep(flush_interval)
Logger.info("Import: Flushing #{length(records)} from #{table_name} buffer")
insert_all(table_name, records)
diff --git a/lib/plausible/imported/google_analytics4.ex b/lib/plausible/imported/google_analytics4.ex
new file mode 100644
index 0000000000..5832421334
--- /dev/null
+++ b/lib/plausible/imported/google_analytics4.ex
@@ -0,0 +1,253 @@
+defmodule Plausible.Imported.GoogleAnalytics4 do
+ @moduledoc """
+ Import implementation for Google Analytics 4.
+ """
+
+ use Plausible.Imported.Importer
+
+ @missing_values ["(none)", "(not set)", "(not provided)", "(other)"]
+
+ @impl true
+ def name(), do: :google_analytics_4
+
+ @impl true
+ def label(), do: "Google Analytics 4"
+
+ @impl true
+ def email_template(), do: "google_analytics_import.html"
+
+ @impl true
+ def parse_args(
+ %{"property" => property, "start_date" => start_date, "end_date" => end_date} = args
+ ) do
+ start_date = Date.from_iso8601!(start_date)
+ end_date = Date.from_iso8601!(end_date)
+ date_range = Date.range(start_date, end_date)
+
+ auth = {
+ Map.fetch!(args, "access_token"),
+ Map.fetch!(args, "refresh_token"),
+ Map.fetch!(args, "token_expires_at")
+ }
+
+ [
+ property: property,
+ date_range: date_range,
+ auth: auth
+ ]
+ end
+
+ @doc """
+ Imports stats from a Google Analytics 4 property to a Plausible site.
+
+ This function fetches Google Analytics 4 reports which are then passed in batches
+ to Clickhouse by the `Plausible.Imported.Buffer` process.
+ """
+ @impl true
+ def import_data(site_import, opts) do
+ date_range = Keyword.fetch!(opts, :date_range)
+ property = Keyword.fetch!(opts, :property)
+ auth = Keyword.fetch!(opts, :auth)
+ flush_interval_ms = Keyword.get(opts, :flush_interval_ms, 1000)
+
+ {:ok, buffer} = Plausible.Imported.Buffer.start_link(flush_interval_ms: flush_interval_ms)
+
+ persist_fn = fn table, rows ->
+ records = from_report(rows, site_import.site_id, site_import.id, table)
+ Plausible.Imported.Buffer.insert_many(buffer, table, records)
+ end
+
+ try do
+ Plausible.Google.GA4.API.import_analytics(date_range, property, auth, persist_fn)
+ after
+ Plausible.Imported.Buffer.flush(buffer)
+ Plausible.Imported.Buffer.stop(buffer)
+ end
+ end
+
+ def from_report(nil, _site_id, _import_id, _metric), do: nil
+
+ def from_report(data, site_id, import_id, table) do
+ Enum.reduce(data, [], fn row, acc ->
+ if Map.get(row.dimensions, "date") in @missing_values do
+ acc
+ else
+ [new_from_report(site_id, import_id, table, row) | acc]
+ end
+ end)
+ end
+
+ defp parse_number(nr) do
+ {float, ""} = Float.parse(nr)
+ round(float)
+ end
+
+ defp new_from_report(site_id, import_id, "imported_visitors", row) do
+ %{
+ site_id: site_id,
+ import_id: import_id,
+ date: get_date(row),
+ visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+ pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
+ bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
+ visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
+ visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
+ }
+ end
+
+ defp new_from_report(site_id, import_id, "imported_sources", row) do
+ %{
+ site_id: site_id,
+ import_id: import_id,
+ date: get_date(row),
+ source: row.dimensions |> Map.fetch!("sessionSource") |> parse_referrer(),
+ utm_medium: row.dimensions |> Map.fetch!("sessionMedium") |> default_if_missing(),
+ utm_campaign: row.dimensions |> Map.fetch!("sessionCampaignName") |> default_if_missing(),
+ utm_content: row.dimensions |> Map.fetch!("sessionManualAdContent") |> default_if_missing(),
+ utm_term: row.dimensions |> Map.fetch!("sessionGoogleAdsKeyword") |> default_if_missing(),
+ visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+ visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
+ bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
+ visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
+ }
+ end
+
+ defp new_from_report(site_id, import_id, "imported_pages", row) do
+ %{
+ site_id: site_id,
+ import_id: import_id,
+ date: get_date(row),
+ hostname: row.dimensions |> Map.fetch!("hostName") |> String.replace_prefix("www.", ""),
+ page: row.dimensions |> Map.fetch!("pagePath") |> URI.parse() |> Map.get(:path),
+ visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+ pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
+ # NOTE: no exits metric in GA4 API currently
+ exits: 0,
+ time_on_page: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
+ }
+ end
+
+ defp new_from_report(site_id, import_id, "imported_entry_pages", row) do
+ %{
+ site_id: site_id,
+ import_id: import_id,
+ date: get_date(row),
+ entry_page: row.dimensions |> Map.fetch!("landingPage"),
+ visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+ entrances: row.metrics |> Map.fetch!("sessions") |> parse_number(),
+ visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
+ bounces: row.metrics |> Map.fetch!("bounces") |> parse_number()
+ }
+ end
+
+ # NOTE: no exit pages metrics in GA4 API available for now
+ # defp new_from_report(site_id, import_id, "imported_exit_pages", row) do
+ # %{
+ # site_id: site_id,
+ # import_id: import_id,
+ # date: get_date(row),
+ # exit_page: Map.fetch!(row.dimensions, "exitPage"),
+ # visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+ # exits: row.metrics |> Map.fetch!("sessions") |> parse_number()
+ # }
+ # end
+
+ defp new_from_report(site_id, import_id, "imported_locations", row) do
+ country_code = row.dimensions |> Map.fetch!("countryId") |> default_if_missing("")
+ city_name = row.dimensions |> Map.fetch!("city") |> default_if_missing("")
+ city_data = Location.get_city(city_name, country_code)
+
+ %{
+ site_id: site_id,
+ import_id: import_id,
+ date: get_date(row),
+ country: country_code,
+ region: row.dimensions |> Map.fetch!("region") |> default_if_missing(""),
+ city: city_data && city_data.id,
+ visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+ visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
+ bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
+ visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
+ }
+ end
+
+ defp new_from_report(site_id, import_id, "imported_devices", row) do
+ %{
+ site_id: site_id,
+ import_id: import_id,
+ date: get_date(row),
+ device: row.dimensions |> Map.fetch!("deviceCategory") |> String.capitalize(),
+ visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+ visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
+ bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
+ visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
+ }
+ end
+
+ @browser_google_to_plausible %{
+ "User-Agent:Opera" => "Opera",
+ "Mozilla Compatible Agent" => "Mobile App",
+ "Android Webview" => "Mobile App",
+ "Android Browser" => "Mobile App",
+ "Safari (in-app)" => "Mobile App",
+ "User-Agent: Mozilla" => "Firefox",
+ "(not set)" => ""
+ }
+
+ defp new_from_report(site_id, import_id, "imported_browsers", row) do
+ browser = Map.fetch!(row.dimensions, "browser")
+
+ %{
+ site_id: site_id,
+ import_id: import_id,
+ date: get_date(row),
+ browser: Map.get(@browser_google_to_plausible, browser, browser),
+ visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+ visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
+ bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
+ visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
+ }
+ end
+
+ @os_google_to_plausible %{
+ "Macintosh" => "Mac",
+ "Linux" => "GNU/Linux",
+ "(not set)" => ""
+ }
+
+ defp new_from_report(site_id, import_id, "imported_operating_systems", row) do
+ os = Map.fetch!(row.dimensions, "operatingSystem")
+
+ %{
+ site_id: site_id,
+ import_id: import_id,
+ date: get_date(row),
+ operating_system: Map.get(@os_google_to_plausible, os, os),
+ visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+ visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
+ bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
+ visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
+ }
+ end
+
+ defp get_date(%{dimensions: %{"date" => date}}) do
+ date
+ |> Timex.parse!("%Y%m%d", :strftime)
+ |> NaiveDateTime.to_date()
+ end
+
+ defp default_if_missing(value, default \\ nil)
+ defp default_if_missing(value, default) when value in @missing_values, do: default
+ defp default_if_missing(value, _default), do: value
+
+ defp parse_referrer(nil), do: nil
+ defp parse_referrer("(direct)"), do: nil
+ defp parse_referrer("google"), do: "Google"
+ defp parse_referrer("bing"), do: "Bing"
+ defp parse_referrer("duckduckgo"), do: "DuckDuckGo"
+
+ defp parse_referrer(ref) do
+ RefInspector.parse("https://" <> ref)
+ |> PlausibleWeb.RefInspector.parse()
+ end
+end
diff --git a/lib/plausible/imported/import_sources.ex b/lib/plausible/imported/import_sources.ex
index bcf1eb8d45..96a149b19b 100644
--- a/lib/plausible/imported/import_sources.ex
+++ b/lib/plausible/imported/import_sources.ex
@@ -4,6 +4,7 @@ defmodule Plausible.Imported.ImportSources do
"""
@sources [
+ Plausible.Imported.GoogleAnalytics4,
Plausible.Imported.UniversalAnalytics,
Plausible.Imported.NoopImporter,
Plausible.Imported.CSVImporter
diff --git a/lib/plausible/imported/universal_analytics.ex b/lib/plausible/imported/universal_analytics.ex
index 1c7a1e093e..1d0dcb2e8a 100644
--- a/lib/plausible/imported/universal_analytics.ex
+++ b/lib/plausible/imported/universal_analytics.ex
@@ -102,7 +102,7 @@ defmodule Plausible.Imported.UniversalAnalytics do
end
try do
- Plausible.Google.Api.import_analytics(date_range, view_id, auth, persist_fn)
+ Plausible.Google.UA.API.import_analytics(date_range, view_id, auth, persist_fn)
after
Plausible.Imported.Buffer.flush(buffer)
Plausible.Imported.Buffer.stop(buffer)
diff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.ex
index 11a68efc52..a242413710 100644
--- a/lib/plausible_web/controllers/auth_controller.ex
+++ b/lib/plausible_web/controllers/auth_controller.ex
@@ -697,13 +697,16 @@ defmodule PlausibleWeb.AuthController do
end
def google_auth_callback(conn, %{"error" => error, "state" => state} = params) do
- [site_id, _redirected_to, legacy] =
+ [site_id, _redirected_to, legacy, _ga4] =
case Jason.decode!(state) do
[site_id, redirect_to] ->
- [site_id, redirect_to, true]
+ [site_id, redirect_to, true, false]
[site_id, redirect_to, legacy] ->
- [site_id, redirect_to, legacy]
+ [site_id, redirect_to, legacy, false]
+
+ [site_id, redirect_to, legacy, ga4] ->
+ [site_id, redirect_to, legacy, ga4]
end
site = Repo.get(Plausible.Site, site_id)
@@ -745,15 +748,18 @@ defmodule PlausibleWeb.AuthController do
end
def google_auth_callback(conn, %{"code" => code, "state" => state}) do
- res = Plausible.Google.HTTP.fetch_access_token(code)
+ res = Plausible.Google.API.fetch_access_token!(code)
- [site_id, redirect_to, legacy] =
+ [site_id, redirect_to, legacy, ga4] =
case Jason.decode!(state) do
[site_id, redirect_to] ->
- [site_id, redirect_to, true]
+ [site_id, redirect_to, true, false]
[site_id, redirect_to, legacy] ->
- [site_id, redirect_to, legacy]
+ [site_id, redirect_to, legacy, false]
+
+ [site_id, redirect_to, legacy, ga4] ->
+ [site_id, redirect_to, legacy, ga4]
end
site = Repo.get(Plausible.Site, site_id)
@@ -761,15 +767,26 @@ defmodule PlausibleWeb.AuthController do
case redirect_to do
"import" ->
- redirect(conn,
- external:
- Routes.site_path(conn, :import_from_google_view_id_form, site.domain,
- access_token: res["access_token"],
- refresh_token: res["refresh_token"],
- expires_at: NaiveDateTime.to_iso8601(expires_at),
- legacy: legacy
- )
- )
+ if ga4 do
+ redirect(conn,
+ external:
+ Routes.google_analytics4_path(conn, :property_form, site.domain,
+ access_token: res["access_token"],
+ refresh_token: res["refresh_token"],
+ expires_at: NaiveDateTime.to_iso8601(expires_at)
+ )
+ )
+ else
+ redirect(conn,
+ external:
+ Routes.universal_analytics_path(conn, :view_id_form, site.domain,
+ access_token: res["access_token"],
+ refresh_token: res["refresh_token"],
+ expires_at: NaiveDateTime.to_iso8601(expires_at),
+ legacy: legacy
+ )
+ )
+ end
_ ->
id_token = res["id_token"]
diff --git a/lib/plausible_web/controllers/google_analytics4_controller.ex b/lib/plausible_web/controllers/google_analytics4_controller.ex
new file mode 100644
index 0000000000..3fd55d3ab6
--- /dev/null
+++ b/lib/plausible_web/controllers/google_analytics4_controller.ex
@@ -0,0 +1,143 @@
+defmodule PlausibleWeb.GoogleAnalytics4Controller do
+ use PlausibleWeb, :controller
+
+ plug(PlausibleWeb.RequireAccountPlug)
+
+ plug(PlausibleWeb.AuthorizeSiteAccess, [:owner, :admin, :super_admin])
+
+ def property_form(conn, %{
+ "access_token" => access_token,
+ "refresh_token" => refresh_token,
+ "expires_at" => expires_at
+ }) do
+ redirect_route = Routes.site_path(conn, :settings_imports_exports, conn.assigns.site.domain)
+
+ case Plausible.Google.GA4.API.list_properties(access_token) do
+ {:ok, properties} ->
+ conn
+ |> assign(:skip_plausible_tracking, true)
+ |> render("property_form.html",
+ access_token: access_token,
+ refresh_token: refresh_token,
+ expires_at: expires_at,
+ site: conn.assigns.site,
+ properties: properties,
+ layout: {PlausibleWeb.LayoutView, "focus.html"}
+ )
+
+ {:error, :authentication_failed} ->
+ conn
+ |> put_flash(
+ :error,
+ "We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again."
+ )
+ |> redirect(external: redirect_route)
+
+ {:error, _any} ->
+ conn
+ |> put_flash(
+ :error,
+ "We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance."
+ )
+ |> redirect(external: redirect_route)
+ end
+ end
+
+ def property(conn, %{
+ "property" => property,
+ "access_token" => access_token,
+ "refresh_token" => refresh_token,
+ "expires_at" => expires_at
+ }) do
+ site = conn.assigns.site
+ start_date = Plausible.Google.GA4.API.get_analytics_start_date(access_token, property)
+
+ case start_date do
+ {:ok, nil} ->
+ {:ok, properties} = Plausible.Google.GA4.API.list_properties(access_token)
+
+ conn
+ |> assign(:skip_plausible_tracking, true)
+ |> render("property_form.html",
+ access_token: access_token,
+ refresh_token: refresh_token,
+ expires_at: expires_at,
+ site: site,
+ properties: properties,
+ selected_property_error: "No data found. Nothing to import",
+ layout: {PlausibleWeb.LayoutView, "focus.html"}
+ )
+
+ {:ok, _date} ->
+ redirect(conn,
+ to:
+ Routes.google_analytics4_path(conn, :confirm, site.domain,
+ property: property,
+ access_token: access_token,
+ refresh_token: refresh_token,
+ expires_at: expires_at
+ )
+ )
+ end
+ end
+
+ def confirm(conn, %{
+ "property" => property,
+ "access_token" => access_token,
+ "refresh_token" => refresh_token,
+ "expires_at" => expires_at
+ }) do
+ site = conn.assigns.site
+
+ start_date = Plausible.Google.GA4.API.get_analytics_start_date(access_token, property)
+
+ end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone)
+
+ {:ok, {property_name, property}} =
+ Plausible.Google.GA4.API.get_property(access_token, property)
+
+ conn
+ |> assign(:skip_plausible_tracking, true)
+ |> render("confirm.html",
+ access_token: access_token,
+ refresh_token: refresh_token,
+ expires_at: expires_at,
+ site: site,
+ selected_property: property,
+ selected_property_name: property_name,
+ start_date: start_date,
+ end_date: end_date,
+ layout: {PlausibleWeb.LayoutView, "focus.html"}
+ )
+ end
+
+ def import(conn, %{
+ "property" => property,
+ "start_date" => start_date,
+ "end_date" => end_date,
+ "access_token" => access_token,
+ "refresh_token" => refresh_token,
+ "expires_at" => expires_at
+ }) do
+ site = conn.assigns.site
+ current_user = conn.assigns.current_user
+
+ redirect_route = Routes.site_path(conn, :settings_imports_exports, site.domain)
+
+ {:ok, _} =
+ Plausible.Imported.GoogleAnalytics4.new_import(
+ site,
+ current_user,
+ property: property,
+ start_date: start_date,
+ end_date: end_date,
+ access_token: access_token,
+ refresh_token: refresh_token,
+ token_expires_at: expires_at
+ )
+
+ conn
+ |> put_flash(:success, "Import scheduled. An email will be sent when it completes.")
+ |> redirect(external: redirect_route)
+ end
+end
diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex
index 51c04c7991..aa38531cc7 100644
--- a/lib/plausible_web/controllers/site_controller.ex
+++ b/lib/plausible_web/controllers/site_controller.ex
@@ -232,7 +232,7 @@ defmodule PlausibleWeb.SiteController do
search_console_domains =
if site.google_auth do
- Plausible.Google.Api.fetch_verified_properties(site.google_auth)
+ Plausible.Google.API.fetch_verified_properties(site.google_auth)
end
imported_pageviews =
@@ -641,198 +641,6 @@ defmodule PlausibleWeb.SiteController do
end
end
- def import_from_google_user_metric_notice(conn, %{
- "view_id" => view_id,
- "access_token" => access_token,
- "refresh_token" => refresh_token,
- "expires_at" => expires_at,
- "legacy" => legacy
- }) do
- site = conn.assigns[:site]
-
- conn
- |> assign(:skip_plausible_tracking, true)
- |> render("import_from_google_user_metric_form.html",
- site: site,
- view_id: view_id,
- access_token: access_token,
- refresh_token: refresh_token,
- expires_at: expires_at,
- legacy: legacy,
- layout: {PlausibleWeb.LayoutView, "focus.html"}
- )
- end
-
- def import_from_google_view_id_form(conn, %{
- "access_token" => access_token,
- "refresh_token" => refresh_token,
- "expires_at" => expires_at,
- "legacy" => legacy
- }) do
- redirect_route =
- if legacy == "true" do
- Routes.site_path(conn, :settings_integrations, conn.assigns.site.domain)
- else
- Routes.site_path(conn, :settings_imports_exports, conn.assigns.site.domain)
- end
-
- case Plausible.Google.Api.list_views(access_token) do
- {:ok, view_ids} ->
- conn
- |> assign(:skip_plausible_tracking, true)
- |> render("import_from_google_view_id_form.html",
- access_token: access_token,
- refresh_token: refresh_token,
- expires_at: expires_at,
- site: conn.assigns.site,
- view_ids: view_ids,
- legacy: legacy,
- layout: {PlausibleWeb.LayoutView, "focus.html"}
- )
-
- {:error, :authentication_failed} ->
- conn
- |> put_flash(
- :error,
- "We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again."
- )
- |> redirect(external: redirect_route)
-
- {:error, _any} ->
- conn
- |> put_flash(
- :error,
- "We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance."
- )
- |> redirect(external: redirect_route)
- end
- end
-
- # see https://stackoverflow.com/a/57416769
- @google_analytics_new_user_metric_date ~D[2016-08-24]
- def import_from_google_view_id(conn, %{
- "view_id" => view_id,
- "access_token" => access_token,
- "refresh_token" => refresh_token,
- "expires_at" => expires_at,
- "legacy" => legacy
- }) do
- site = conn.assigns[:site]
- start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token)
-
- case start_date do
- {:ok, nil} ->
- site = conn.assigns[:site]
- {:ok, view_ids} = Plausible.Google.Api.list_views(access_token)
-
- conn
- |> assign(:skip_plausible_tracking, true)
- |> render("import_from_google_view_id_form.html",
- access_token: access_token,
- refresh_token: refresh_token,
- expires_at: expires_at,
- site: site,
- view_ids: view_ids,
- selected_view_id_error: "No data found. Nothing to import",
- legacy: legacy,
- layout: {PlausibleWeb.LayoutView, "focus.html"}
- )
-
- {:ok, date} ->
- if Timex.before?(date, @google_analytics_new_user_metric_date) do
- redirect(conn,
- to:
- Routes.site_path(conn, :import_from_google_user_metric_notice, site.domain,
- view_id: view_id,
- access_token: access_token,
- refresh_token: refresh_token,
- expires_at: expires_at,
- legacy: legacy
- )
- )
- else
- redirect(conn,
- to:
- Routes.site_path(conn, :import_from_google_confirm, site.domain,
- view_id: view_id,
- access_token: access_token,
- refresh_token: refresh_token,
- expires_at: expires_at,
- legacy: legacy
- )
- )
- end
- end
- end
-
- def import_from_google_confirm(conn, %{
- "view_id" => view_id,
- "access_token" => access_token,
- "refresh_token" => refresh_token,
- "expires_at" => expires_at,
- "legacy" => legacy
- }) do
- site = conn.assigns[:site]
-
- start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token)
-
- end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone)
-
- {:ok, {view_name, view_id}} = Plausible.Google.Api.get_view(access_token, view_id)
-
- conn
- |> assign(:skip_plausible_tracking, true)
- |> render("import_from_google_confirm.html",
- access_token: access_token,
- refresh_token: refresh_token,
- expires_at: expires_at,
- site: site,
- selected_view_id: view_id,
- selected_view_id_name: view_name,
- start_date: start_date,
- end_date: end_date,
- legacy: legacy,
- layout: {PlausibleWeb.LayoutView, "focus.html"}
- )
- end
-
- def import_from_google(conn, %{
- "view_id" => view_id,
- "start_date" => start_date,
- "end_date" => end_date,
- "access_token" => access_token,
- "refresh_token" => refresh_token,
- "expires_at" => expires_at,
- "legacy" => legacy
- }) do
- site = conn.assigns.site
- current_user = conn.assigns.current_user
-
- redirect_route =
- if legacy == "true" do
- Routes.site_path(conn, :settings_integrations, site.domain)
- else
- Routes.site_path(conn, :settings_imports_exports, site.domain)
- end
-
- {:ok, _} =
- Plausible.Imported.UniversalAnalytics.new_import(
- site,
- current_user,
- view_id: view_id,
- start_date: start_date,
- end_date: end_date,
- access_token: access_token,
- refresh_token: refresh_token,
- token_expires_at: expires_at,
- legacy: legacy == "true"
- )
-
- conn
- |> put_flash(:success, "Import scheduled. An email will be sent when it completes.")
- |> redirect(external: redirect_route)
- end
-
def forget_import(conn, %{"import_id" => import_id}) do
site = conn.assigns.site
diff --git a/lib/plausible_web/controllers/universal_analytics_controller.ex b/lib/plausible_web/controllers/universal_analytics_controller.ex
new file mode 100644
index 0000000000..16a4bea64e
--- /dev/null
+++ b/lib/plausible_web/controllers/universal_analytics_controller.ex
@@ -0,0 +1,198 @@
+defmodule PlausibleWeb.UniversalAnalyticsController do
+ use PlausibleWeb, :controller
+
+ plug(PlausibleWeb.RequireAccountPlug)
+
+ plug(PlausibleWeb.AuthorizeSiteAccess, [:owner, :admin, :super_admin])
+
+ def user_metric_notice(conn, %{
+ "view_id" => view_id,
+ "access_token" => access_token,
+ "refresh_token" => refresh_token,
+ "expires_at" => expires_at,
+ "legacy" => legacy
+ }) do
+ site = conn.assigns.site
+
+ conn
+ |> assign(:skip_plausible_tracking, true)
+ |> render("user_metric_form.html",
+ site: site,
+ view_id: view_id,
+ access_token: access_token,
+ refresh_token: refresh_token,
+ expires_at: expires_at,
+ legacy: legacy,
+ layout: {PlausibleWeb.LayoutView, "focus.html"}
+ )
+ end
+
+ def view_id_form(conn, %{
+ "access_token" => access_token,
+ "refresh_token" => refresh_token,
+ "expires_at" => expires_at,
+ "legacy" => legacy
+ }) do
+ redirect_route =
+ if legacy == "true" do
+ Routes.site_path(conn, :settings_integrations, conn.assigns.site.domain)
+ else
+ Routes.site_path(conn, :settings_imports_exports, conn.assigns.site.domain)
+ end
+
+ case Plausible.Google.UA.API.list_views(access_token) do
+ {:ok, view_ids} ->
+ conn
+ |> assign(:skip_plausible_tracking, true)
+ |> render("view_id_form.html",
+ access_token: access_token,
+ refresh_token: refresh_token,
+ expires_at: expires_at,
+ site: conn.assigns.site,
+ view_ids: view_ids,
+ legacy: legacy,
+ layout: {PlausibleWeb.LayoutView, "focus.html"}
+ )
+
+ {:error, :authentication_failed} ->
+ conn
+ |> put_flash(
+ :error,
+ "We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again."
+ )
+ |> redirect(external: redirect_route)
+
+ {:error, _any} ->
+ conn
+ |> put_flash(
+ :error,
+ "We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance."
+ )
+ |> redirect(external: redirect_route)
+ end
+ end
+
+ # see https://stackoverflow.com/a/57416769
+ @google_analytics_new_user_metric_date ~D[2016-08-24]
+ def view_id(conn, %{
+ "view_id" => view_id,
+ "access_token" => access_token,
+ "refresh_token" => refresh_token,
+ "expires_at" => expires_at,
+ "legacy" => legacy
+ }) do
+ site = conn.assigns.site
+ start_date = Plausible.Google.UA.API.get_analytics_start_date(access_token, view_id)
+
+ case start_date do
+ {:ok, nil} ->
+ {:ok, view_ids} = Plausible.Google.UA.API.list_views(access_token)
+
+ conn
+ |> assign(:skip_plausible_tracking, true)
+ |> render("view_id_form.html",
+ access_token: access_token,
+ refresh_token: refresh_token,
+ expires_at: expires_at,
+ site: site,
+ view_ids: view_ids,
+ selected_view_id_error: "No data found. Nothing to import",
+ legacy: legacy,
+ layout: {PlausibleWeb.LayoutView, "focus.html"}
+ )
+
+ {:ok, date} ->
+ if Timex.before?(date, @google_analytics_new_user_metric_date) do
+ redirect(conn,
+ to:
+ Routes.universal_analytics_path(conn, :user_metric_notice, site.domain,
+ view_id: view_id,
+ access_token: access_token,
+ refresh_token: refresh_token,
+ expires_at: expires_at,
+ legacy: legacy
+ )
+ )
+ else
+ redirect(conn,
+ to:
+ Routes.universal_analytics_path(conn, :confirm, site.domain,
+ view_id: view_id,
+ access_token: access_token,
+ refresh_token: refresh_token,
+ expires_at: expires_at,
+ legacy: legacy
+ )
+ )
+ end
+ end
+ end
+
+ def confirm(conn, %{
+ "view_id" => view_id,
+ "access_token" => access_token,
+ "refresh_token" => refresh_token,
+ "expires_at" => expires_at,
+ "legacy" => legacy
+ }) do
+ site = conn.assigns.site
+
+ start_date = Plausible.Google.UA.API.get_analytics_start_date(access_token, view_id)
+
+ end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone)
+
+ {:ok, {view_name, view_id}} = Plausible.Google.UA.API.get_view(access_token, view_id)
+
+ conn
+ |> assign(:skip_plausible_tracking, true)
+ |> render("confirm.html",
+ access_token: access_token,
+ refresh_token: refresh_token,
+ expires_at: expires_at,
+ site: site,
+ selected_view_id: view_id,
+ selected_view_id_name: view_name,
+ start_date: start_date,
+ end_date: end_date,
+ legacy: legacy,
+ layout: {PlausibleWeb.LayoutView, "focus.html"}
+ )
+ end
+
+ def import(conn, %{
+ "view_id" => view_id,
+ "start_date" => start_date,
+ "end_date" => end_date,
+ "access_token" => access_token,
+ "refresh_token" => refresh_token,
+ "expires_at" => expires_at,
+ "legacy" => legacy
+ }) do
+ site = conn.assigns.site
+ current_user = conn.assigns.current_user
+
+ redirect_route =
+ if legacy == "true" do
+ Routes.site_path(conn, :settings_integrations, site.domain)
+ else
+ Routes.site_path(conn, :settings_imports_exports, site.domain)
+ end
+
+ {:ok, _} =
+ Plausible.Imported.UniversalAnalytics.new_import(
+ site,
+ current_user,
+ view_id: view_id,
+ start_date: start_date,
+ end_date: end_date,
+ access_token: access_token,
+ refresh_token: refresh_token,
+ token_expires_at: expires_at,
+ legacy: legacy == "true"
+ )
+
+ conn
+ |> put_flash(:success, "Import scheduled. An email will be sent when it completes.")
+ |> redirect(external: redirect_route)
+ end
+end
diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex
index d1c414ac76..162dee3cad 100644
--- a/lib/plausible_web/router.ex
+++ b/lib/plausible_web/router.ex
@@ -374,17 +374,27 @@ defmodule PlausibleWeb.Router do
delete "/:website/stats", SiteController, :reset_stats
get "/:website/import/google-analytics/view-id",
- SiteController,
- :import_from_google_view_id_form
+ UniversalAnalyticsController,
+ :view_id_form
- post "/:website/import/google-analytics/view-id", SiteController, :import_from_google_view_id
+ post "/:website/import/google-analytics/view-id", UniversalAnalyticsController, :view_id
get "/:website/import/google-analytics/user-metric",
- SiteController,
- :import_from_google_user_metric_notice
+ UniversalAnalyticsController,
+ :user_metric_notice
+
+ get "/:website/import/google-analytics/confirm", UniversalAnalyticsController, :confirm
+ post "/:website/settings/google-import", UniversalAnalyticsController, :import
+
+ get "/:website/import/google-analytics4/property",
+ GoogleAnalytics4Controller,
+ :property_form
+
+ post "/:website/import/google-analytics4/property", GoogleAnalytics4Controller, :property
+
+ get "/:website/import/google-analytics4/confirm", GoogleAnalytics4Controller, :confirm
+ post "/:website/settings/google4-import", GoogleAnalytics4Controller, :import
- get "/:website/import/google-analytics/confirm", SiteController, :import_from_google_confirm
- post "/:website/settings/google-import", SiteController, :import_from_google
delete "/:website/settings/forget-imported", SiteController, :forget_imported
delete "/:website/settings/forget-import/:import_id", SiteController, :forget_import
diff --git a/lib/plausible_web/templates/google_analytics4/confirm.html.heex b/lib/plausible_web/templates/google_analytics4/confirm.html.heex
new file mode 100644
index 0000000000..b0f63048ba
--- /dev/null
+++ b/lib/plausible_web/templates/google_analytics4/confirm.html.heex
@@ -0,0 +1,46 @@
+<%= form_for @conn, Routes.google_analytics4_path(@conn, :import, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>
+ <h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>
+
+ <%= hidden_input(f, :access_token, value: @access_token) %>
+ <%= hidden_input(f, :refresh_token, value: @refresh_token) %>
+ <%= hidden_input(f, :expires_at, value: @expires_at) %>
+
+ <%= case @start_date do %>
+ <% {:ok, start_date} -> %>
+ <div class="mt-6 text-sm text-gray-500 dark:text-gray-200">
+ Stats from this property and time period will be imported from your Google Analytics 4 account to your Plausible dashboard
+ </div>
+
+ <div class="mt-6">
+ <%= styled_label(f, :property, "Google Analytics 4 property") %>
+ <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
+ <%= @selected_property_name %>
+ </span>
+ <%= hidden_input(f, :property, readonly: "true", value: @selected_property) %>
+ </div>
+ <div class="flex justify-between mt-3">
+ <div class="w-36">
+ <%= styled_label(f, :start_date, "From") %>
+ <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
+ <%= PlausibleWeb.EmailView.date_format(start_date) %>
+ </span>
+ <%= hidden_input(f, :start_date, value: start_date, readonly: "true") %>
+ </div>
+ <div class="align-middle pt-4 dark:text-gray-100">&rarr;</div>
+ <div class="w-36">
+ <%= styled_label(f, :end_date, "To") %>
+ <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
+ <%= PlausibleWeb.EmailView.date_format(@end_date) %>
+ </span>
+ <%= hidden_input(f, :end_date, value: @end_date, readonly: "true") %>
+ </div>
+ </div>
+ <% {:error, error} -> %>
+ <p class="text-gray-700 dark:text-gray-300 mt-6">
+ The following error occurred when fetching your Google Analytics 4 data.
+ </p>
+ <p class="text-red-700 font-medium mt-3"><%= error %></p>
+ <% end %>
+
+ <%= submit("Confirm import", class: "button mt-6") %>
+<% end %>
diff --git a/lib/plausible_web/templates/google_analytics4/property_form.html.heex b/lib/plausible_web/templates/google_analytics4/property_form.html.heex
new file mode 100644
index 0000000000..c88461073c
--- /dev/null
+++ b/lib/plausible_web/templates/google_analytics4/property_form.html.heex
@@ -0,0 +1,19 @@
+<%= form_for @conn, Routes.google_analytics4_path(@conn, :property, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>
+ <h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics 4</h2>
+
+ <%= hidden_input(f, :access_token, value: @access_token) %>
+ <%= hidden_input(f, :refresh_token, value: @refresh_token) %>
+ <%= hidden_input(f, :expires_at, value: @expires_at) %>
+
+ <div class="mt-6 text-sm text-gray-500 dark:text-gray-200">
+ Choose the property in your Google Analytics 4 account that will be imported to the <%= @site.domain %> dashboard.
+ </div>
+
+ <div class="mt-3">
+ <%= styled_label(f, :property, "Google Analytics 4 property") %>
+ <%= styled_select(f, :property, @properties, prompt: "(Choose property)", required: "true") %>
+ <%= styled_error(@conn.assigns[:selected_property_error]) %>
+ </div>
+
+ <%= submit("Continue ->", class: "button mt-6") %>
+<% end %>
diff --git a/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex b/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex
deleted file mode 100644
index e5792ed9f9..0000000000
--- a/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex
+++ /dev/null
@@ -1,26 +0,0 @@
-<div class="max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8">
- <h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>
-
- <div class="mt-6 text-sm text-gray-500 dark:text-gray-200">
- <p>
- <svg xmlns="http://www.w3.org/2000/svg" class="h-4 w-4 inline text-orange-600" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
- <path stroke-linecap="round" stroke-linejoin="round" d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-3L13.732 4c-.77-1.333-2.694-1.333-3.464 0L3.34 16c-.77 1.333.192 3 1.732 3z" />
- </svg>
-
- Important: Since your GA property includes data from before 23rd August 2016, you have to take an extra step to make sure we can import data smoothly.
- </p>
-
- <ol class="mt-4">
- <li>1. Navigate to the GA property you want to import from</li>
- <li>2. Go to Admin &gt; Property Settings &gt; User Analysis</li>
- <li>3. Make sure <i>Enable Users Metric in Reporting</i> is <b>OFF</b></li>
- </ol>
-
- <p class="mt-4">
- The setting may take a few minutes to take effect. If your imported data is showing 0 visitors in unexpected places, it's probably caused by this and you
- can try importing again later.
- </p>
- </div>
-
- <%= link("Continue ->", to: Routes.site_path(@conn, :import_from_google_confirm, @site.domain, view_id: @view_id, access_token: @access_token, refresh_token: @refresh_token, expires_at: @expires_at, legacy: @legacy), class: "button mt-6") %>
-</div>
diff --git a/lib/plausible_web/templates/site/settings_google_import.html.heex b/lib/plausible_web/templates/site/settings_google_import.html.heex
index fb223f2982..c7d76b99c8 100644
--- a/lib/plausible_web/templates/site/settings_google_import.html.heex
+++ b/lib/plausible_web/templates/site/settings_google_import.html.heex
@@ -92,7 +92,7 @@
<% end %>
<PlausibleWeb.Components.Google.button
id="analytics-connect"
- to={Plausible.Google.Api.import_authorize_url(@site.id, "import", true)}
+ to={Plausible.Google.API.import_authorize_url(@site.id, "import", legacy: true)}
/>
<% end %>
<% else %>
diff --git a/lib/plausible_web/templates/site/settings_imports_exports.html.heex b/lib/plausible_web/templates/site/settings_imports_exports.html.heex
index 9369143a17..b1b31f877c 100644
--- a/lib/plausible_web/templates/site/settings_imports_exports.html.heex
+++ b/lib/plausible_web/templates/site/settings_imports_exports.html.heex
@@ -14,20 +14,22 @@
<PlausibleWeb.Components.Generic.button_link
class="w-36 h-20"
theme="bright"
- href={Plausible.Google.Api.import_authorize_url(@site.id, "import", false)}
+ href={Plausible.Google.API.import_authorize_url(@site.id, "import", legacy: false)}
>
<img src="/images/icon/universal_analytics_logo.svg" alt="New Universal Analytics import" />
</PlausibleWeb.Components.Generic.button_link>
<PlausibleWeb.Components.Generic.button_link
- class="w-36 h-20 opacity-40 cursor-not-allowed"
+ class="w-36 h-20"
theme="bright"
- href=""
+ href={
+ Plausible.Google.API.import_authorize_url(@site.id, "import", legacy: false, ga4: true)
+ }
>
<img
src="/images/icon/google_analytics_4_logo.svg"
width="110"
- alt="New Universal Analytics import"
+ alt="New Google Analytics 4 import"
/>
</PlausibleWeb.Components.Generic.button_link>
diff --git a/lib/plausible_web/templates/site/settings_search_console.html.heex b/lib/plausible_web/templates/site/settings_search_console.html.heex
index 5b5b6aff9c..c615fab3aa 100644
--- a/lib/plausible_web/templates/site/settings_search_console.html.heex
+++ b/lib/plausible_web/templates/site/settings_search_console.html.heex
@@ -82,7 +82,7 @@
<% else %>
<PlausibleWeb.Components.Google.button
id="search-console-connect"
- to={Plausible.Google.Api.search_console_authorize_url(@site.id, "search-console")}
+ to={Plausible.Google.API.search_console_authorize_url(@site.id, "search-console")}
/>
<div class="text-gray-700 dark:text-gray-300 mt-8">
NB: You also need to set up your site on
diff --git a/lib/plausible_web/templates/site/import_from_google_confirm.html.eex b/lib/plausible_web/templates/universal_analytics/confirm.html.heex
similarity index 52%
rename from lib/plausible_web/templates/site/import_from_google_confirm.html.eex
rename to lib/plausible_web/templates/universal_analytics/confirm.html.heex
index c46459aa94..3d6f143f32 100644
--- a/lib/plausible_web/templates/site/import_from_google_confirm.html.eex
+++ b/lib/plausible_web/templates/universal_analytics/confirm.html.heex
@@ -1,4 +1,4 @@
-<%= form_for @conn, Routes.site_path(@conn, :import_from_google, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>
+<%= form_for @conn, Routes.universal_analytics_path(@conn, :import, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>
<h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>
<%= hidden_input(f, :access_token, value: @access_token) %>
@@ -8,33 +8,40 @@
<%= case @start_date do %>
<% {:ok, start_date} -> %>
-
<div class="mt-6 text-sm text-gray-500 dark:text-gray-200">
Stats from this property and time period will be imported from your Google Analytics account to your Plausible dashboard
</div>
<div class="mt-6">
<%= styled_label(f, :view_id, "Google Analytics view") %>
- <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800"><%= @selected_view_id_name %></span>
- <%= hidden_input f, :view_id, readonly: "true", value: @selected_view_id %>
+ <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
+ <%= @selected_view_id_name %>
+ </span>
+ <%= hidden_input(f, :view_id, readonly: "true", value: @selected_view_id) %>
</div>
<div class="flex justify-between mt-3">
<div class="w-36">
- <%= styled_label f, :start_date, "From" %>
- <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800"><%= PlausibleWeb.EmailView.date_format(start_date) %></span>
- <%= hidden_input f, :start_date, value: start_date, readonly: "true" %>
+ <%= styled_label(f, :start_date, "From") %>
+ <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
+ <%= PlausibleWeb.EmailView.date_format(start_date) %>
+ </span>
+ <%= hidden_input(f, :start_date, value: start_date, readonly: "true") %>
</div>
<div class="align-middle pt-4 dark:text-gray-100">&rarr;</div>
<div class="w-36">
- <%= styled_label f, :end_date, "To" %>
- <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800"><%= PlausibleWeb.EmailView.date_format(@end_date) %></span>
- <%= hidden_input f, :end_date, value: @end_date, readonly: "true" %>
+ <%= styled_label(f, :end_date, "To") %>
+ <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">
+ <%= PlausibleWeb.EmailView.date_format(@end_date) %>
+ </span>
+ <%= hidden_input(f, :end_date, value: @end_date, readonly: "true") %>
</div>
</div>
<% {:error, error} -> %>
- <p class="text-gray-700 dark:text-gray-300 mt-6">The following error occurred when fetching your Google Analytics data.</p>
+ <p class="text-gray-700 dark:text-gray-300 mt-6">
+ The following error occurred when fetching your Google Analytics data.
+ </p>
<p class="text-red-700 font-medium mt-3"><%= error %></p>
- <% end %>
+ <% end %>
- <%= submit "Confirm import", class: "button mt-6" %>
+ <%= submit("Confirm import", class: "button mt-6") %>
<% end %>
diff --git a/lib/plausible_web/templates/universal_analytics/user_metric_form.html.heex b/lib/plausible_web/templates/universal_analytics/user_metric_form.html.heex
new file mode 100644
index 0000000000..2f353f984b
--- /dev/null
+++ b/lib/plausible_web/templates/universal_analytics/user_metric_form.html.heex
@@ -0,0 +1,46 @@
+<div class="max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8">
+ <h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>
+
+ <div class="mt-6 text-sm text-gray-500 dark:text-gray-200">
+ <p>
+ <svg
+ xmlns="http://www.w3.org/2000/svg"
+ class="h-4 w-4 inline text-orange-600"
+ fill="none"
+ viewBox="0 0 24 24"
+ stroke="currentColor"
+ stroke-width="2"
+ >
+ <path
+ stroke-linecap="round"
+ stroke-linejoin="round"
+ d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-3L13.732 4c-.77-1.333-2.694-1.333-3.464 0L3.34 16c-.77 1.333.192 3 1.732 3z"
+ />
+ </svg>
+ Important: Since your GA property includes data from before 23rd August 2016, you have to take an extra step to make sure we can import data smoothly.
+ </p>
+
+ <ol class="mt-4">
+ <li>1. Navigate to the GA property you want to import from</li>
+ <li>2. Go to Admin &gt; Property Settings &gt; User Analysis</li>
+ <li>3. Make sure <i>Enable Users Metric in Reporting</i> is <b>OFF</b></li>
+ </ol>
+
+ <p class="mt-4">
+ The setting may take a few minutes to take effect. If your imported data is showing 0 visitors in unexpected places, it's probably caused by this and you
+ can try importing again later.
+ </p>
+ </div>
+
+ <%= link("Continue ->",
+ to:
+ Routes.universal_analytics_path(@conn, :confirm, @site.domain,
+ view_id: @view_id,
+ access_token: @access_token,
+ refresh_token: @refresh_token,
+ expires_at: @expires_at,
+ legacy: @legacy
+ ),
+ class: "button mt-6"
+ ) %>
+</div>
diff --git a/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex b/lib/plausible_web/templates/universal_analytics/view_id_form.html.heex
similarity index 66%
rename from lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex
rename to lib/plausible_web/templates/universal_analytics/view_id_form.html.heex
index d8b8db4442..e9e371f75c 100644
--- a/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex
+++ b/lib/plausible_web/templates/universal_analytics/view_id_form.html.heex
@@ -1,4 +1,4 @@
-<%= form_for @conn, Routes.site_path(@conn, :import_from_google_view_id, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>
+<%= form_for @conn, Routes.universal_analytics_path(@conn, :view_id, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>
<h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>
<%= hidden_input(f, :access_token, value: @access_token) %>
@@ -12,9 +12,9 @@
<div class="mt-3">
<%= styled_label(f, :view_id, "Google Analytics view") %>
- <%= styled_select f, :view_id, @view_ids, prompt: "(Choose view)", required: "true" %>
+ <%= styled_select(f, :view_id, @view_ids, prompt: "(Choose view)", required: "true") %>
<%= styled_error(@conn.assigns[:selected_view_id_error]) %>
</div>
- <%= submit "Continue ->", class: "button mt-6" %>
+ <%= submit("Continue ->", class: "button mt-6") %>
<% end %>
diff --git a/lib/plausible_web/views/google_analytics4_view.ex b/lib/plausible_web/views/google_analytics4_view.ex
new file mode 100644
index 0000000000..760309b804
--- /dev/null
+++ b/lib/plausible_web/views/google_analytics4_view.ex
@@ -0,0 +1,4 @@
+defmodule PlausibleWeb.GoogleAnalytics4View do
+ use PlausibleWeb, :view
+ use Plausible
+end
diff --git a/lib/plausible_web/views/universal_analytics_view.ex b/lib/plausible_web/views/universal_analytics_view.ex
new file mode 100644
index 0000000000..f95899ee77
--- /dev/null
+++ b/lib/plausible_web/views/universal_analytics_view.ex
@@ -0,0 +1,4 @@
+defmodule PlausibleWeb.UniversalAnalyticsView do
+ use PlausibleWeb, :view
+ use Plausible
+end
diff --git a/test/plausible/google/api_test.exs b/test/plausible/google/api_test.exs
index c448e7c431..076a1af469 100644
--- a/test/plausible/google/api_test.exs
+++ b/test/plausible/google/api_test.exs
@@ -1,8 +1,8 @@
-defmodule Plausible.Google.ApiTest do
+defmodule Plausible.Google.APITest do
use Plausible.DataCase, async: true
use Plausible.Test.Support.HTTPMocker
- alias Plausible.Google.Api
+ alias Plausible.Google
alias Plausible.Imported.UniversalAnalytics
import ExUnit.CaptureLog
@@ -44,7 +44,7 @@ defmodule Plausible.Google.ApiTest do
Plausible.Imported.Buffer.insert_many(buffer, table, records)
end
- assert :ok == Plausible.Google.Api.import_analytics(date_range, view_id, auth, persist_fn)
+ assert :ok == Google.UA.API.import_analytics(date_range, view_id, auth, persist_fn)
Plausible.Imported.Buffer.flush(buffer)
Plausible.Imported.Buffer.stop(buffer)
@@ -86,7 +86,7 @@ defmodule Plausible.Google.ApiTest do
Plausible.Imported.Buffer.insert_many(buffer, table, records)
end
- assert :ok == Plausible.Google.Api.import_analytics(range, "123551", auth, persist_fn)
+ assert :ok == Google.UA.API.import_analytics(range, "123551", auth, persist_fn)
Plausible.Imported.Buffer.flush(buffer)
Plausible.Imported.Buffer.stop(buffer)
@@ -98,7 +98,7 @@ defmodule Plausible.Google.ApiTest do
@tag :slow
test "will fetch and persist import data from Google Analytics" do
- request = %Plausible.Google.ReportRequest{
+ request = %Plausible.Google.UA.ReportRequest{
dataset: "imported_exit_pages",
view_id: "123",
date_range: Date.range(~D[2022-01-01], ~D[2022-02-01]),
@@ -127,7 +127,7 @@ defmodule Plausible.Google.ApiTest do
hideValueRanges: true,
metrics: [%{expression: "ga:users"}, %{expression: "ga:exits"}],
orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}],
- pageSize: 10000,
+ pageSize: 10_000,
pageToken: nil,
viewId: "123"
}
@@ -139,7 +139,7 @@ defmodule Plausible.Google.ApiTest do
)
assert :ok =
- Api.fetch_and_persist(request,
+ Google.UA.API.fetch_and_persist(request,
sleep_time: 0,
persist_fn: fn dataset, row ->
assert dataset == "imported_exit_pages"
@@ -167,7 +167,7 @@ defmodule Plausible.Google.ApiTest do
end
)
- request = %Plausible.Google.ReportRequest{
+ request = %Plausible.Google.UA.ReportRequest{
view_id: "123",
date_range: Date.range(~D[2022-01-01], ~D[2022-02-01]),
dimensions: ["ga:date"],
@@ -178,7 +178,7 @@ defmodule Plausible.Google.ApiTest do
}
assert {:error, :request_failed} =
- Api.fetch_and_persist(request,
+ Google.UA.API.fetch_and_persist(request,
sleep_time: 0,
persist_fn: fn _dataset, _rows -> :ok end
)
@@ -197,7 +197,7 @@ defmodule Plausible.Google.ApiTest do
end
)
- request = %Plausible.Google.ReportRequest{
+ request = %Plausible.Google.UA.ReportRequest{
dataset: "imported_exit_pages",
view_id: "123",
date_range: Date.range(~D[2022-01-01], ~D[2022-02-01]),
@@ -209,7 +209,7 @@ defmodule Plausible.Google.ApiTest do
}
assert :ok ==
- Api.fetch_and_persist(request,
+ Google.UA.API.fetch_and_persist(request,
sleep_time: 0,
persist_fn: fn dataset, rows ->
assert dataset == "imported_exit_pages"
@@ -253,7 +253,7 @@ defmodule Plausible.Google.ApiTest do
query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}
- assert {:error, "google_auth_error"} = Plausible.Google.Api.fetch_stats(site, query, 5)
+ assert {:error, "google_auth_error"} = Google.API.fetch_stats(site, query, 5)
end
test "returns whatever error code google returns on API client error", %{site: site} do
@@ -270,7 +270,7 @@ defmodule Plausible.Google.ApiTest do
query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}
- assert {:error, "some_error"} = Plausible.Google.Api.fetch_stats(site, query, 5)
+ assert {:error, "some_error"} = Google.API.fetch_stats(site, query, 5)
end
test "returns generic HTTP error and logs it", %{site: site} do
@@ -290,7 +290,7 @@ defmodule Plausible.Google.ApiTest do
log =
capture_log(fn ->
assert {:error, "failed_to_list_stats"} =
- Plausible.Google.Api.fetch_stats(site, query, 5)
+ Google.API.fetch_stats(site, query, 5)
end)
assert log =~ "Google Analytics: failed to list stats: %Finch.Error{reason: :some_reason}"
@@ -314,7 +314,7 @@ defmodule Plausible.Google.ApiTest do
[
%{name: ["keyword1", "keyword2"], visitors: 25},
%{name: ["keyword3", "keyword4"], visitors: 15}
- ]} = Plausible.Google.Api.fetch_stats(site, query, 5)
+ ]} = Google.API.fetch_stats(site, query, 5)
end
test "returns next page when page argument is set", %{user: user, site: site} do
@@ -336,7 +336,7 @@ defmodule Plausible.Google.ApiTest do
[
%{name: ["keyword1", "keyword2"], visitors: 25},
%{name: ["keyword3", "keyword4"], visitors: 15}
- ]} = Plausible.Google.Api.fetch_stats(site, query, 5)
+ ]} = Google.API.fetch_stats(site, query, 5)
end
test "defaults first page when page argument is not set", %{user: user, site: site} do
@@ -355,7 +355,7 @@ defmodule Plausible.Google.ApiTest do
[
%{name: ["keyword1", "keyword2"], visitors: 25},
%{name: ["keyword3", "keyword4"], visitors: 15}
- ]} = Plausible.Google.Api.fetch_stats(site, query, 5)
+ ]} = Google.API.fetch_stats(site, query, 5)
end
test "returns error when token refresh fails", %{user: user, site: site} do
@@ -372,7 +372,7 @@ defmodule Plausible.Google.ApiTest do
query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}
- assert {:error, "invalid_grant"} = Plausible.Google.Api.fetch_stats(site, query, 5)
+ assert {:error, "invalid_grant"} = Google.API.fetch_stats(site, query, 5)
end
end
@@ -393,7 +393,7 @@ defmodule Plausible.Google.ApiTest do
%{
"one.test" => [{"57238190 - one.test", "57238190"}],
"two.test" => [{"54460083 - two.test", "54460083"}]
- }} == Plausible.Google.Api.list_views("access_token")
+ }} == Google.UA.API.list_views("access_token")
end
test "list_views/1 returns authentication_failed when request fails with HTTP 403" do
@@ -405,7 +405,7 @@ defmodule Plausible.Google.ApiTest do
end
)
- assert {:error, :authentication_failed} == Plausible.Google.Api.list_views("access_token")
+ assert {:error, :authentication_failed} == Google.UA.API.list_views("access_token")
end
test "list_views/1 returns authentication_failed when request fails with HTTP 401" do
@@ -417,7 +417,7 @@ defmodule Plausible.Google.ApiTest do
end
)
- assert {:error, :authentication_failed} == Plausible.Google.Api.list_views("access_token")
+ assert {:error, :authentication_failed} == Google.UA.API.list_views("access_token")
end
test "list_views/1 returns error when request fails with HTTP 500" do
@@ -429,6 +429,6 @@ defmodule Plausible.Google.ApiTest do
end
)
- assert {:error, :unknown} == Plausible.Google.Api.list_views("access_token")
+ assert {:error, :unknown} == Google.UA.API.list_views("access_token")
end
end
diff --git a/test/plausible/google/ga4/api_test.exs b/test/plausible/google/ga4/api_test.exs
new file mode 100644
index 0000000000..029c46ff7c
--- /dev/null
+++ b/test/plausible/google/ga4/api_test.exs
@@ -0,0 +1,57 @@
+defmodule Plausible.Google.GA4.APITest do
+ use Plausible.DataCase, async: true
+
+ import Mox
+
+ alias Plausible.Google.GA4
+
+ setup :verify_on_exit!
+
+ describe "list_properties/1" do
+ test "returns list of properties grouped by accounts" do
+ result = Jason.decode!(File.read!("fixture/ga4_list_properties.json"))
+
+ expect(Plausible.HTTPClient.Mock, :get, fn _url, _opts ->
+ {:ok, %Finch.Response{status: 200, body: result}}
+ end)
+
+ assert {:ok, accounts} = GA4.API.list_properties("some_access_token")
+
+ assert [
+ {"account.one (accounts/28425178)",
+ [{"account.one - GA4 (properties/428685906)", "properties/428685906"}]},
+ {"Demo Account (accounts/54516992)",
+ [
+ {"GA4 - Flood-It! (properties/153293282)", "properties/153293282"},
+ {"GA4 - Google Merch Shop (properties/213025502)", "properties/213025502"}
+ ]}
+ ] = accounts
+ end
+ end
+
+ describe "get_property/2" do
+ test "returns tuple consisting of display name and value of a property" do
+ result = Jason.decode!(File.read!("fixture/ga4_list_properties.json"))
+
+ expect(Plausible.HTTPClient.Mock, :get, fn _url, _opts ->
+ {:ok, %Finch.Response{status: 200, body: result}}
+ end)
+
+ assert {:ok, {"GA4 - Flood-It! (properties/153293282)", "properties/153293282"}} =
+ GA4.API.get_property("some_access_token", "properties/153293282")
+ end
+ end
+
+ describe "get_analytics_start_date/2" do
+ test "returns stats start date for a given property" do
+ result = Jason.decode!(File.read!("fixture/ga4_start_date.json"))
+
+ expect(Plausible.HTTPClient.Mock, :post, fn _url, _headers, _body ->
+ {:ok, %Finch.Response{status: 200, body: result}}
+ end)
+
+ assert {:ok, ~D[2024-02-22]} =
+ GA4.API.get_analytics_start_date("some_access_token", "properties/153293282")
+ end
+ end
+end
diff --git a/test/plausible/imported/google_analytics4_test.exs b/test/plausible/imported/google_analytics4_test.exs
new file mode 100644
index 0000000000..20e0384405
--- /dev/null
+++ b/test/plausible/imported/google_analytics4_test.exs
@@ -0,0 +1,98 @@
+defmodule Plausible.Imported.GoogleAnalytics4Test do
+ use Plausible.DataCase, async: true
+
+ import Mox
+
+ import Ecto.Query, only: [from: 2]
+
+ alias Plausible.Imported.GoogleAnalytics4
+
+ @refresh_token_body Jason.decode!(File.read!("fixture/ga_refresh_token.json"))
+
+ @full_report_mock [
+ "fixture/ga4_report_imported_visitors.json",
+ "fixture/ga4_report_imported_sources.json",
+ "fixture/ga4_report_imported_pages.json",
+ "fixture/ga4_report_imported_entry_pages.json",
+ "fixture/ga4_report_imported_locations.json",
+ "fixture/ga4_report_imported_devices.json",
+ "fixture/ga4_report_imported_browsers.json",
+ "fixture/ga4_report_imported_operating_systems.json"
+ ]
+ |> Enum.map(&File.read!/1)
+ |> Enum.map(&Jason.decode!/1)
+
+ setup :verify_on_exit!
+
+ describe "parse_args/1 and import_data/2" do
+ setup [:create_user, :create_new_site]
+
+ test "imports data returned from GA4 Data API", %{user: user, site: site} do
+ past = DateTime.add(DateTime.utc_now(), -3600, :second)
+
+ {:ok, job} =
+ Plausible.Imported.GoogleAnalytics4.new_import(
+ site,
+ user,
+ property: "properties/123456",
+ start_date: ~D[2024-02-20],
+ end_date: Date.utc_today(),
+ access_token: "redacted_access_token",
+ refresh_token: "redacted_refresh_token",
+ token_expires_at: DateTime.to_iso8601(past)
+ )
+
+ site_import = Plausible.Imported.get_import(job.args.import_id)
+
+ opts = job |> Repo.reload!() |> Map.get(:args) |> GoogleAnalytics4.parse_args()
+
+ opts = Keyword.put(opts, :flush_interval_ms, 10)
+
+ expect(Plausible.HTTPClient.Mock, :post, fn "https://www.googleapis.com/oauth2/v4/token",
+ headers,
+ body ->
+ assert [{"content-type", "application/x-www-form-urlencoded"}] == headers
+
+ assert %{
+ grant_type: :refresh_token,
+ redirect_uri: "http://localhost:8000/auth/google/callback",
+ refresh_token: "redacted_refresh_token"
+ } = body
+
+ {:ok, %Finch.Response{status: 200, body: @refresh_token_body}}
+ end)
+
+ for report <- @full_report_mock do
+ expect(Plausible.HTTPClient.Mock, :post, fn _url, headers, _body, _opts ->
+ assert [{"Authorization", "Bearer 1/fFAGRNJru1FTz70BzhT3Zg"}] == headers
+ {:ok, %Finch.Response{status: 200, body: report}}
+ end)
+ end
+
+ Enum.each(Plausible.Imported.tables(), fn table ->
+ query = from(imported in table, where: imported.site_id == ^site.id)
+ assert await_clickhouse_count(query, 0)
+ end)
+
+ assert :ok = GoogleAnalytics4.import_data(site_import, opts)
+
+ Enum.each(Plausible.Imported.tables(), fn table ->
+ count =
+ case table do
+ "imported_sources" -> 3
+ "imported_visitors" -> 3
+ "imported_pages" -> 8
+ "imported_entry_pages" -> 4
+ "imported_exit_pages" -> 0
+ "imported_locations" -> 4
+ "imported_devices" -> 4
+ "imported_browsers" -> 5
+ "imported_operating_systems" -> 4
+ end
+
+ query = from(imported in table, where: imported.site_id == ^site.id)
+ assert await_clickhouse_count(query, count)
+ end)
+ end
+ end
+end
diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs
index d26395638e..45f1c02381 100644
--- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs
@@ -1461,7 +1461,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do
])
conn = get(conn, "/api/stats/#{site.domain}/referrers/Google?period=day")
- {:ok, terms} = Plausible.Google.Api.Mock.fetch_stats(nil, nil, nil)
+ {:ok, terms} = Plausible.Google.API.Mock.fetch_stats(nil, nil, nil)
assert json_response(conn, 200) == %{
"total_visitors" => 2,
diff --git a/test/support/google_api_mock.ex b/test/support/google_api_mock.ex
index 5aa33a7180..878cbec970 100644
--- a/test/support/google_api_mock.ex
+++ b/test/support/google_api_mock.ex
@@ -1,4 +1,8 @@
-defmodule Plausible.Google.Api.Mock do
+defmodule Plausible.Google.API.Mock do
+ @moduledoc """
+ Mock of API to Google services.
+ """
+
def fetch_stats(_auth, _query, _limit) do
{:ok,
[