Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Implement basics of GA4 import (#3851)
Adrian Gruntkowski4d7d88cfeccb
diff --git a/config/config.exs b/config/config.exsindex b3a1f05e88..f7f2f32d80 100644--- a/config/config.exs+++ b/config/config.exs@@ -45,7 +45,7 @@ config :ref_inspector,config :plausible,paddle_api: Plausible.Billing.PaddleApi,- google_api: Plausible.Google.Api+ google_api: Plausible.Google.APIconfig :plausible,# 30 minutesdiff --git a/config/test.exs b/config/test.exsindex 31182b2846..d1d9edfac0 100644--- a/config/test.exs+++ b/config/test.exs@@ -16,7 +16,7 @@ config :plausible, Plausible.Mailer, adapter: Bamboo.TestAdapterconfig :plausible,paddle_api: Plausible.PaddleApi.Mock,- google_api: Plausible.Google.Api.Mock+ google_api: Plausible.Google.API.Mockconfig :bamboo, :refute_timeout, 10diff --git a/fixture/ga4_list_properties.json b/fixture/ga4_list_properties.jsonnew file mode 100644index 0000000000..f8aac0b1d6--- /dev/null+++ b/fixture/ga4_list_properties.json@@ -0,0 +1,41 @@+{+ "accountSummaries": [+ {+ "account": "accounts/28425178",+ "displayName": "account.one",+ "name": "accountSummaries/28425178",+ "propertySummaries": [+ {+ "displayName": "account.one - GA4",+ "parent": "accounts/28425178",+ "property": "properties/428685906",+ "propertyType": "PROPERTY_TYPE_ORDINARY"+ }+ ]+ },+ {+ "account": "accounts/45336102",+ "displayName": "account.two",+ "name": "accountSummaries/45336102"+ },+ {+ "account": "accounts/54516992",+ "displayName": "Demo Account",+ "name": "accountSummaries/54516992",+ "propertySummaries": [+ {+ "displayName": "GA4 - Flood-It!",+ "parent": "accounts/54516992",+ "property": "properties/153293282",+ "propertyType": "PROPERTY_TYPE_ORDINARY"+ },+ {+ "displayName": "GA4 - Google Merch Shop",+ "parent": "accounts/54516992",+ "property": "properties/213025502",+ "propertyType": "PROPERTY_TYPE_ORDINARY"+ }+ ]+ }+ ]+}\ No newline at end of filediff --git a/fixture/ga4_report_imported_browsers.json b/fixture/ga4_report_imported_browsers.jsonnew file mode 100644index 0000000000..214ac2b268--- /dev/null+++ b/fixture/ga4_report_imported_browsers.json@@ -0,0 +1,161 @@+{+ "kind": "analyticsData#batchRunReports",+ "reports": [+ {+ "dimensionHeaders": [+ {+ "name": "date"+ },+ {+ "name": "browser"+ }+ ],+ "kind": "analyticsData#runReport",+ "metadata": {+ "currencyCode": "USD",+ "timeZone": "Europe/Warsaw"+ },+ "metricHeaders": [+ {+ "name": "totalUsers",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "sessions",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "bounces",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "userEngagementDuration",+ "type": "TYPE_SECONDS"+ }+ ],+ "rowCount": 5,+ "rows": [+ {+ "dimensionValues": [+ {+ "value": "20240226"+ },+ {+ "value": "Safari"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "0"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240224"+ },+ {+ "value": "Chrome"+ }+ ],+ "metricValues": [+ {+ "value": "2"+ },+ {+ "value": "2"+ },+ {+ "value": "2"+ },+ {+ "value": "0"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240222"+ },+ {+ "value": "Chrome"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "0"+ },+ {+ "value": "4"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240222"+ },+ {+ "value": "Firefox"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "2"+ },+ {+ "value": "0"+ },+ {+ "value": "21"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240222"+ },+ {+ "value": "Safari"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "0"+ },+ {+ "value": "4"+ }+ ]+ }+ ]+ }+ ]+}\ No newline at end of filediff --git a/fixture/ga4_report_imported_devices.json b/fixture/ga4_report_imported_devices.jsonnew file mode 100644index 0000000000..e8bcc01047--- /dev/null+++ b/fixture/ga4_report_imported_devices.json@@ -0,0 +1,137 @@+{+ "kind": "analyticsData#batchRunReports",+ "reports": [+ {+ "dimensionHeaders": [+ {+ "name": "date"+ },+ {+ "name": "deviceCategory"+ }+ ],+ "kind": "analyticsData#runReport",+ "metadata": {+ "currencyCode": "USD",+ "timeZone": "Europe/Warsaw"+ },+ "metricHeaders": [+ {+ "name": "totalUsers",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "sessions",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "bounces",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "userEngagementDuration",+ "type": "TYPE_SECONDS"+ }+ ],+ "rowCount": 4,+ "rows": [+ {+ "dimensionValues": [+ {+ "value": "20240226"+ },+ {+ "value": "mobile"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "0"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240224"+ },+ {+ "value": "desktop"+ }+ ],+ "metricValues": [+ {+ "value": "2"+ },+ {+ "value": "2"+ },+ {+ "value": "2"+ },+ {+ "value": "0"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240222"+ },+ {+ "value": "desktop"+ }+ ],+ "metricValues": [+ {+ "value": "2"+ },+ {+ "value": "3"+ },+ {+ "value": "0"+ },+ {+ "value": "25"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240222"+ },+ {+ "value": "mobile"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "0"+ },+ {+ "value": "4"+ }+ ]+ }+ ]+ }+ ]+}\ No newline at end of filediff --git a/fixture/ga4_report_imported_entry_pages.json b/fixture/ga4_report_imported_entry_pages.jsonnew file mode 100644index 0000000000..77370bb30f--- /dev/null+++ b/fixture/ga4_report_imported_entry_pages.json@@ -0,0 +1,137 @@+{+ "kind": "analyticsData#batchRunReports",+ "reports": [+ {+ "dimensionHeaders": [+ {+ "name": "date"+ },+ {+ "name": "landingPage"+ }+ ],+ "kind": "analyticsData#runReport",+ "metadata": {+ "currencyCode": "USD",+ "timeZone": "Europe/Warsaw"+ },+ "metricHeaders": [+ {+ "name": "totalUsers",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "sessions",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "userEngagementDuration",+ "type": "TYPE_SECONDS"+ },+ {+ "name": "bounces",+ "type": "TYPE_INTEGER"+ }+ ],+ "rowCount": 4,+ "rows": [+ {+ "dimensionValues": [+ {+ "value": "20240226"+ },+ {+ "value": "/blog/firstpost"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "0"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240224"+ },+ {+ "value": "/"+ }+ ],+ "metricValues": [+ {+ "value": "2"+ },+ {+ "value": "2"+ },+ {+ "value": "0"+ },+ {+ "value": "2"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240222"+ },+ {+ "value": "/"+ }+ ],+ "metricValues": [+ {+ "value": "2"+ },+ {+ "value": "3"+ },+ {+ "value": "25"+ },+ {+ "value": "0"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240222"+ },+ {+ "value": "/blog/unicode-in-elixir"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "4"+ },+ {+ "value": "0"+ }+ ]+ }+ ]+ }+ ]+}\ No newline at end of filediff --git a/fixture/ga4_report_imported_locations.json b/fixture/ga4_report_imported_locations.jsonnew file mode 100644index 0000000000..b731a21025--- /dev/null+++ b/fixture/ga4_report_imported_locations.json@@ -0,0 +1,167 @@+{+ "kind": "analyticsData#batchRunReports",+ "reports": [+ {+ "dimensionHeaders": [+ {+ "name": "date"+ },+ {+ "name": "countryId"+ },+ {+ "name": "region"+ },+ {+ "name": "city"+ }+ ],+ "kind": "analyticsData#runReport",+ "metadata": {+ "currencyCode": "USD",+ "timeZone": "Europe/Warsaw"+ },+ "metricHeaders": [+ {+ "name": "totalUsers",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "sessions",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "bounces",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "userEngagementDuration",+ "type": "TYPE_SECONDS"+ }+ ],+ "rowCount": 4,+ "rows": [+ {+ "dimensionValues": [+ {+ "value": "20240226"+ },+ {+ "value": "PL"+ },+ {+ "value": "Masovian Voivodeship"+ },+ {+ "value": "Warsaw"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "0"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240224"+ },+ {+ "value": "US"+ },+ {+ "value": "California"+ },+ {+ "value": "(not set)"+ }+ ],+ "metricValues": [+ {+ "value": "2"+ },+ {+ "value": "2"+ },+ {+ "value": "2"+ },+ {+ "value": "0"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240222"+ },+ {+ "value": "PL"+ },+ {+ "value": "Pomeranian Voivodeship"+ },+ {+ "value": "Gdansk"+ }+ ],+ "metricValues": [+ {+ "value": "2"+ },+ {+ "value": "3"+ },+ {+ "value": "0"+ },+ {+ "value": "25"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240222"+ },+ {+ "value": "ES"+ },+ {+ "value": "Catalonia"+ },+ {+ "value": "Barcelona"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "0"+ },+ {+ "value": "4"+ }+ ]+ }+ ]+ }+ ]+}\ No newline at end of filediff --git a/fixture/ga4_report_imported_operating_systems.json b/fixture/ga4_report_imported_operating_systems.jsonnew file mode 100644index 0000000000..158fe1efa9--- /dev/null+++ b/fixture/ga4_report_imported_operating_systems.json@@ -0,0 +1,137 @@+{+ "kind": "analyticsData#batchRunReports",+ "reports": [+ {+ "dimensionHeaders": [+ {+ "name": "date"+ },+ {+ "name": "operatingSystem"+ }+ ],+ "kind": "analyticsData#runReport",+ "metadata": {+ "currencyCode": "USD",+ "timeZone": "Europe/Warsaw"+ },+ "metricHeaders": [+ {+ "name": "totalUsers",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "sessions",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "bounces",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "userEngagementDuration",+ "type": "TYPE_SECONDS"+ }+ ],+ "rowCount": 4,+ "rows": [+ {+ "dimensionValues": [+ {+ "value": "20240226"+ },+ {+ "value": "iOS"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "0"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240224"+ },+ {+ "value": "Windows"+ }+ ],+ "metricValues": [+ {+ "value": "2"+ },+ {+ "value": "2"+ },+ {+ "value": "2"+ },+ {+ "value": "0"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240222"+ },+ {+ "value": "Macintosh"+ }+ ],+ "metricValues": [+ {+ "value": "2"+ },+ {+ "value": "3"+ },+ {+ "value": "0"+ },+ {+ "value": "25"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240222"+ },+ {+ "value": "iOS"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "0"+ },+ {+ "value": "4"+ }+ ]+ }+ ]+ }+ ]+}\ No newline at end of filediff --git a/fixture/ga4_report_imported_pages.json b/fixture/ga4_report_imported_pages.jsonnew file mode 100644index 0000000000..f7be04f265--- /dev/null+++ b/fixture/ga4_report_imported_pages.json@@ -0,0 +1,232 @@+{+ "kind": "analyticsData#batchRunReports",+ "reports": [+ {+ "dimensionHeaders": [+ {+ "name": "date"+ },+ {+ "name": "hostName"+ },+ {+ "name": "pagePath"+ }+ ],+ "kind": "analyticsData#runReport",+ "metadata": {+ "currencyCode": "USD",+ "timeZone": "Europe/Warsaw"+ },+ "metricHeaders": [+ {+ "name": "totalUsers",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "screenPageViews",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "userEngagementDuration",+ "type": "TYPE_SECONDS"+ }+ ],+ "rowCount": 8,+ "rows": [+ {+ "dimensionValues": [+ {+ "value": "20240226"+ },+ {+ "value": "drawer.todo.computer"+ },+ {+ "value": "/blog/firstpost/"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "0"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240224"+ },+ {+ "value": "drawer-4l3.pages.dev"+ },+ {+ "value": "/"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "0"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240224"+ },+ {+ "value": "drawer.todo.computer"+ },+ {+ "value": "/"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "0"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240222"+ },+ {+ "value": "drawer.todo.computer"+ },+ {+ "value": "/"+ }+ ],+ "metricValues": [+ {+ "value": "2"+ },+ {+ "value": "3"+ },+ {+ "value": "7"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240222"+ },+ {+ "value": "drawer.todo.computer"+ },+ {+ "value": "/blog/unicode-in-elixir/"+ }+ ],+ "metricValues": [+ {+ "value": "2"+ },+ {+ "value": "7"+ },+ {+ "value": "21"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240222"+ },+ {+ "value": "drawer.todo.computer"+ },+ {+ "value": "/about/"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "0"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240222"+ },+ {+ "value": "drawer.todo.computer"+ },+ {+ "value": "/blog/"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "1"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240222"+ },+ {+ "value": "drawer.todo.computer"+ },+ {+ "value": "/blog/firstpost/"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "0"+ }+ ]+ }+ ]+ }+ ]+}\ No newline at end of filediff --git a/fixture/ga4_report_imported_sources.json b/fixture/ga4_report_imported_sources.jsonnew file mode 100644index 0000000000..4c5edb5179--- /dev/null+++ b/fixture/ga4_report_imported_sources.json@@ -0,0 +1,161 @@+{+ "kind": "analyticsData#batchRunReports",+ "reports": [+ {+ "dimensionHeaders": [+ {+ "name": "date"+ },+ {+ "name": "sessionSource"+ },+ {+ "name": "sessionMedium"+ },+ {+ "name": "sessionCampaignName"+ },+ {+ "name": "sessionManualAdContent"+ },+ {+ "name": "sessionGoogleAdsKeyword"+ }+ ],+ "kind": "analyticsData#runReport",+ "metadata": {+ "currencyCode": "USD",+ "timeZone": "Europe/Warsaw"+ },+ "metricHeaders": [+ {+ "name": "totalUsers",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "sessions",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "bounces",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "userEngagementDuration",+ "type": "TYPE_SECONDS"+ }+ ],+ "rowCount": 3,+ "rows": [+ {+ "dimensionValues": [+ {+ "value": "20240226"+ },+ {+ "value": "(direct)"+ },+ {+ "value": "(none)"+ },+ {+ "value": "(direct)"+ },+ {+ "value": "(not set)"+ },+ {+ "value": "(not set)"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "0"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240224"+ },+ {+ "value": "(direct)"+ },+ {+ "value": "(none)"+ },+ {+ "value": "(direct)"+ },+ {+ "value": "(not set)"+ },+ {+ "value": "(not set)"+ }+ ],+ "metricValues": [+ {+ "value": "2"+ },+ {+ "value": "2"+ },+ {+ "value": "2"+ },+ {+ "value": "0"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240222"+ },+ {+ "value": "(direct)"+ },+ {+ "value": "(none)"+ },+ {+ "value": "(direct)"+ },+ {+ "value": "(not set)"+ },+ {+ "value": "(not set)"+ }+ ],+ "metricValues": [+ {+ "value": "3"+ },+ {+ "value": "4"+ },+ {+ "value": "0"+ },+ {+ "value": "29"+ }+ ]+ }+ ]+ }+ ]+}\ No newline at end of filediff --git a/fixture/ga4_report_imported_visitors.json b/fixture/ga4_report_imported_visitors.jsonnew file mode 100644index 0000000000..78d29b5e70--- /dev/null+++ b/fixture/ga4_report_imported_visitors.json@@ -0,0 +1,114 @@+{+ "kind": "analyticsData#batchRunReports",+ "reports": [+ {+ "dimensionHeaders": [+ {+ "name": "date"+ }+ ],+ "kind": "analyticsData#runReport",+ "metadata": {+ "currencyCode": "USD",+ "timeZone": "Europe/Warsaw"+ },+ "metricHeaders": [+ {+ "name": "totalUsers",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "screenPageViews",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "bounces",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "sessions",+ "type": "TYPE_INTEGER"+ },+ {+ "name": "userEngagementDuration",+ "type": "TYPE_SECONDS"+ }+ ],+ "rowCount": 3,+ "rows": [+ {+ "dimensionValues": [+ {+ "value": "20240226"+ }+ ],+ "metricValues": [+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "1"+ },+ {+ "value": "0"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240224"+ }+ ],+ "metricValues": [+ {+ "value": "2"+ },+ {+ "value": "2"+ },+ {+ "value": "2"+ },+ {+ "value": "2"+ },+ {+ "value": "0"+ }+ ]+ },+ {+ "dimensionValues": [+ {+ "value": "20240222"+ }+ ],+ "metricValues": [+ {+ "value": "3"+ },+ {+ "value": "13"+ },+ {+ "value": "0"+ },+ {+ "value": "4"+ },+ {+ "value": "29"+ }+ ]+ }+ ]+ }+ ]+}\ No newline at end of filediff --git a/fixture/ga4_start_date.json b/fixture/ga4_start_date.jsonnew file mode 100644index 0000000000..6c1d410d23--- /dev/null+++ b/fixture/ga4_start_date.json@@ -0,0 +1,38 @@+{+ "kind": "analyticsData#batchRunReports",+ "reports": [+ {+ "dimensionHeaders": [+ {+ "name": "date"+ }+ ],+ "kind": "analyticsData#runReport",+ "metadata": {+ "currencyCode": "USD",+ "timeZone": "Europe/Warsaw"+ },+ "metricHeaders": [+ {+ "name": "screenPageViews",+ "type": "TYPE_INTEGER"+ }+ ],+ "rowCount": 3,+ "rows": [+ {+ "dimensionValues": [+ {+ "value": "20240222"+ }+ ],+ "metricValues": [+ {+ "value": "13"+ }+ ]+ }+ ]+ }+ ]+}\ No newline at end of filediff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.exindex 085513886d..6b1e44b4ea 100644--- a/lib/plausible/google/api.ex+++ b/lib/plausible/google/api.ex@@ -1,9 +1,13 @@-defmodule Plausible.Google.Api do- alias Plausible.Google.{ReportRequest, HTTP}+defmodule Plausible.Google.API do+ @moduledoc """+ API to Google services.+ """+use Timex- require Logger- @type google_analytics_view() :: {view_name :: String.t(), view_id :: String.t()}+ alias Plausible.Google.HTTP++ require Logger@search_console_scope URI.encode_www_form("email https://www.googleapis.com/auth/webmasters.readonly"@@ -17,9 +21,16 @@ defmodule Plausible.Google.Api doJason.encode!([site_id, redirect_to])end- def import_authorize_url(site_id, redirect_to, legacy \\ true) do+ def import_authorize_url(site_id, redirect_to, opts \\ []) do+ legacy = Keyword.get(opts, :legacy, true)+ ga4 = Keyword.get(opts, :ga4, false)+"https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@import_scope}&state=" <>- Jason.encode!([site_id, redirect_to, legacy])+ Jason.encode!([site_id, redirect_to, legacy, ga4])+ end++ def fetch_access_token!(code) do+ HTTP.fetch_access_token!(code)enddef fetch_verified_properties(auth) do@@ -53,138 +64,7 @@ defmodule Plausible.Google.Api doendend- @spec list_views(access_token :: String.t()) ::- {:ok, %{(hostname :: String.t()) => [google_analytics_view()]}} | {:error, term()}- @doc """- Lists Google Analytics views grouped by hostname.- """- def list_views(access_token) do- case HTTP.list_views_for_user(access_token) do- {:ok, %{"items" => views}} ->- views = Enum.group_by(views, &view_hostname/1, &view_names/1)- {:ok, views}-- error ->- error- end- end-- defp view_hostname(view) do- case view do- %{"websiteUrl" => url} when is_binary(url) -> url |> URI.parse() |> Map.get(:host)- _any -> "Others"- end- end-- defp view_names(%{"name" => name, "id" => id}) do- {"#{id} - #{name}", id}- end-- @spec get_view(access_token :: String.t(), lookup_id :: String.t()) ::- {:ok, google_analytics_view()} | {:ok, nil} | {:error, term()}- @doc """- Returns a single Google Analytics view if the user has access to it.- """- def get_view(access_token, lookup_id) do- case list_views(access_token) do- {:ok, views} ->- view =- views- |> Map.values()- |> List.flatten()- |> Enum.find(fn {_name, id} -> id == lookup_id end)-- {:ok, view}-- {:error, cause} ->- {:error, cause}- end- end-- @type import_auth :: {- access_token :: String.t(),- refresh_token :: String.t(),- expires_at :: String.t()- }-- @per_page 7_500- @backoff_factor :timer.seconds(10)- @max_attempts 5- @spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) ::- :ok | {:error, term()}- @doc """- Imports stats from a Google Analytics UA view to a Plausible site.-- This function fetches Google Analytics reports in batches of #{@per_page} per- request. The batches are then passed to persist callback.-- Requests to Google Analytics can fail, and are retried at most- #{@max_attempts} times with an exponential backoff. Returns `:ok` when- importing has finished or `{:error, term()}` when a request to GA failed too- many times.-- Useful links:-- - [Feature documentation](https://plausible.io/docs/google-analytics-import)- - [GA API reference](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest)- - [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer)-- """- def import_analytics(date_range, view_id, auth, persist_fn) do- with {:ok, access_token} <- maybe_refresh_token(auth) do- do_import_analytics(date_range, view_id, access_token, persist_fn)- end- end-- defp do_import_analytics(date_range, view_id, access_token, persist_fn) do- Enum.reduce_while(ReportRequest.full_report(), :ok, fn report_request, :ok ->- report_request = %ReportRequest{- report_request- | date_range: date_range,- view_id: view_id,- access_token: access_token,- page_token: nil,- page_size: @per_page- }-- case fetch_and_persist(report_request, persist_fn: persist_fn) do- :ok -> {:cont, :ok}- {:error, _} = error -> {:halt, error}- end- end)- end-- @spec fetch_and_persist(ReportRequest.t(), Keyword.t()) ::- :ok | {:error, term()}- def fetch_and_persist(%ReportRequest{} = report_request, opts \\ []) do- persist_fn = Keyword.fetch!(opts, :persist_fn)- attempt = Keyword.get(opts, :attempt, 1)- sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)-- case HTTP.get_report(report_request) do- {:ok, {rows, next_page_token}} ->- :ok = persist_fn.(report_request.dataset, rows)-- if next_page_token do- fetch_and_persist(- %ReportRequest{report_request | page_token: next_page_token},- opts- )- else- :ok- end-- {:error, cause} ->- if attempt >= @max_attempts do- {:error, cause}- else- Process.sleep(attempt * sleep_time)- fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))- end- end- end-- defp maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do+ def maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) dowith true <- needs_to_refresh_token?(auth.expires),{:ok, {new_access_token, expires_at}} <- do_refresh_token(auth.refresh_token),changeset <-@@ -200,7 +80,7 @@ defmodule Plausible.Google.Api doendend- defp maybe_refresh_token({access_token, refresh_token, expires_at}) do+ def maybe_refresh_token({access_token, refresh_token, expires_at}) dowith true <- needs_to_refresh_token?(expires_at),{:ok, {new_access_token, _expires_at}} <- do_refresh_token(refresh_token) do{:ok, new_access_token}diff --git a/lib/plausible/google/ga4/api.ex b/lib/plausible/google/ga4/api.exnew file mode 100644index 0000000000..f9062d00e3--- /dev/null+++ b/lib/plausible/google/ga4/api.ex@@ -0,0 +1,142 @@+defmodule Plausible.Google.GA4.API do+ @moduledoc """+ API for Google Analytics 4.+ """++ alias Plausible.Google+ alias Plausible.Google.GA4++ require Logger++ @type import_auth :: {+ access_token :: String.t(),+ refresh_token :: String.t(),+ expires_at :: String.t()+ }++ @per_page 50_000+ @backoff_factor :timer.seconds(10)+ @max_attempts 5++ def list_properties(access_token) do+ case GA4.HTTP.list_accounts_for_user(access_token) do+ {:ok, %{"accountSummaries" => accounts}} ->+ accounts =+ accounts+ |> Enum.filter(& &1["propertySummaries"])+ |> Enum.map(fn account ->+ {"#{account["displayName"]} (#{account["account"]})",+ Enum.map(account["propertySummaries"], fn property ->+ {"#{property["displayName"]} (#{property["property"]})", property["property"]}+ end)}+ end)++ {:ok, accounts}++ error ->+ error+ end+ end++ def get_property(access_token, lookup_property) do+ case list_properties(access_token) do+ {:ok, properties} ->+ property =+ properties+ |> Enum.map(&elem(&1, 1))+ |> List.flatten()+ |> Enum.find(fn {_name, property} -> property == lookup_property end)++ {:ok, property}++ {:error, cause} ->+ {:error, cause}+ end+ end++ def get_analytics_start_date(access_token, property) do+ GA4.HTTP.get_analytics_start_date(access_token, property)+ end++ def import_analytics(date_range, property, auth, persist_fn) do+ Logger.debug(+ "[#{inspect(__MODULE__)}:#{property}] Starting import from #{date_range.first} to #{date_range.last}"+ )++ with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do+ do_import_analytics(date_range, property, access_token, persist_fn)+ end+ end++ defp do_import_analytics(date_range, property, access_token, persist_fn) do+ Enum.reduce_while(GA4.ReportRequest.full_report(), :ok, fn report_request, :ok ->+ Logger.debug(+ "[#{inspect(__MODULE__)}:#{property}] Starting to import #{report_request.dataset}"+ )++ report_request = prepare_request(report_request, date_range, property, access_token)++ case fetch_and_persist(report_request, persist_fn: persist_fn) do+ :ok -> {:cont, :ok}+ {:error, _} = error -> {:halt, error}+ end+ end)+ end++ @spec fetch_and_persist(GA4.ReportRequest.t(), Keyword.t()) ::+ :ok | {:error, term()}+ def fetch_and_persist(%GA4.ReportRequest{} = report_request, opts \\ []) do+ persist_fn = Keyword.fetch!(opts, :persist_fn)+ attempt = Keyword.get(opts, :attempt, 1)+ sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)++ case GA4.HTTP.get_report(report_request) do+ {:ok, {rows, row_count}} ->+ Logger.debug(+ "[#{inspect(__MODULE__)}:#{report_request.property}] Fetched #{length(rows)} rows of total #{row_count} with offset #{report_request.offset} for #{report_request.dataset}"+ )++ :ok = persist_fn.(report_request.dataset, rows)++ Logger.debug(+ "[#{inspect(__MODULE__)}:#{report_request.property}] Persisted #{length(rows)} for #{report_request.dataset}"+ )++ if report_request.offset + @per_page < row_count do+ fetch_and_persist(+ %GA4.ReportRequest{report_request | offset: report_request.offset + @per_page},+ opts+ )+ else+ :ok+ end++ {:error, cause} ->+ if attempt >= @max_attempts do+ Logger.debug(+ "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Terminating."+ )++ {:error, cause}+ else+ Logger.debug(+ "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Will retry."+ )++ Process.sleep(attempt * sleep_time)+ fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))+ end+ end+ end++ defp prepare_request(report_request, date_range, property, access_token) do+ %GA4.ReportRequest{+ report_request+ | date_range: date_range,+ property: property,+ access_token: access_token,+ offset: 0,+ limit: @per_page+ }+ end+enddiff --git a/lib/plausible/google/ga4/http.ex b/lib/plausible/google/ga4/http.exnew file mode 100644index 0000000000..da9ace19b2--- /dev/null+++ b/lib/plausible/google/ga4/http.ex@@ -0,0 +1,199 @@+defmodule Plausible.Google.GA4.HTTP do+ @moduledoc """+ HTTP client implementation for Google Analytics 4 API.+ """++ alias Plausible.HTTPClient++ require Logger++ @spec get_report(Plausible.Google.GA4.ReportRequest.t()) ::+ {:ok, {[map()], non_neg_integer()}} | {:error, any()}+ def get_report(%Plausible.Google.GA4.ReportRequest{} = report_request) do+ params = %{+ requests: [+ %{+ property: report_request.property,+ dateRanges: [+ %{+ startDate: report_request.date_range.first,+ endDate: report_request.date_range.last+ }+ ],+ dimensions: Enum.map(report_request.dimensions, &%{name: &1}),+ metrics: Enum.map(report_request.metrics, &build_metric/1),+ orderBys: [+ %{+ dimension: %{+ dimensionName: "date",+ orderType: "ALPHANUMERIC"+ },+ desc: true+ }+ ],+ limit: report_request.limit,+ offset: report_request.offset+ }+ ]+ }++ url =+ "#{reporting_api_url()}/v1beta/#{report_request.property}:batchRunReports"++ response =+ HTTPClient.impl().post(+ url,+ [{"Authorization", "Bearer #{report_request.access_token}"}],+ params,+ receive_timeout: 60_000+ )++ with {:ok, %{body: body}} <- response,+ # File.write!("fixture/ga4_report_#{report_request.dataset}.json", Jason.encode!(body)),+ {:ok, report} <- parse_report_from_response(body),+ row_count <- Map.get(report, "rowCount"),+ {:ok, report} <- convert_to_maps(report) do+ {:ok, {report, row_count}}+ else+ {:error, %{reason: %{status: status, body: body}}} ->+ Logger.debug(+ "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} with code #{status}: #{inspect(body)}"+ )++ Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})+ {:error, :request_failed}++ {:error, reason} ->+ Logger.debug(+ "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}: #{inspect(reason)}"+ )++ {:error, :request_failed}+ end+ end++ defp build_metric(expression) do+ case String.split(expression, " = ") do+ [name, expression] ->+ %{+ name: name,+ expression: expression+ }++ [name] ->+ %{name: name}+ end+ end++ defp parse_report_from_response(%{"reports" => [report | _]}) do+ {:ok, report}+ end++ defp parse_report_from_response(body) do+ Sentry.Context.set_extra_context(%{google_analytics4_response: body})++ Logger.error(+ "Google Analytics 4: Failed to find report in response. Reason: #{inspect(body)}"+ )++ {:error, {:invalid_response, body}}+ end++ defp convert_to_maps(%{+ "rows" => rows,+ "dimensionHeaders" => dimension_headers,+ "metricHeaders" => metric_headers+ })+ when is_list(rows) do+ dimension_headers = Enum.map(dimension_headers, & &1["name"])+ metric_headers = Enum.map(metric_headers, & &1["name"])++ report =+ Enum.map(rows, fn %{"dimensionValues" => dimensions, "metricValues" => metrics} ->+ dimension_values = Enum.map(dimensions, & &1["value"])+ metric_values = Enum.map(metrics, & &1["value"])+ metrics = Enum.zip(metric_headers, metric_values)+ dimensions = Enum.zip(dimension_headers, dimension_values)+ %{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}+ end)++ {:ok, report}+ end++ defp convert_to_maps(response) do+ Logger.error(+ "Google Analytics 4: Failed to read report in response. Reason: #{inspect(response)}"+ )++ Sentry.Context.set_extra_context(%{google_analytics4_response: response})+ {:error, {:invalid_response, response}}+ end++ def list_accounts_for_user(access_token) do+ url = "#{admin_api_url()}/v1beta/accountSummaries"++ headers = [{"Authorization", "Bearer #{access_token}"}]++ case HTTPClient.impl().get(url, headers) do+ {:ok, %Finch.Response{body: body, status: 200}} ->+ {:ok, body}++ {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->+ {:error, :authentication_failed}++ {:error, %HTTPClient.Non200Error{} = error} ->+ Sentry.capture_message("Error listing Google accounts for user", extra: %{error: error})+ {:error, :unknown}+ end+ end++ @earliest_valid_date "2015-08-14"+ def get_analytics_start_date(access_token, property) do+ params = %{+ requests: [+ %{+ property: "#{property}",+ dateRanges: [+ %{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}+ ],+ dimensions: [%{name: "date"}],+ metrics: [%{name: "screenPageViews"}],+ orderBys: [+ %{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: false}+ ],+ limit: 1+ }+ ]+ }++ url = "#{reporting_api_url()}/v1beta/#{property}:batchRunReports"+ headers = [{"Authorization", "Bearer #{access_token}"}]++ case HTTPClient.impl().post(url, headers, params) do+ {:ok, %Finch.Response{body: body, status: 200}} ->+ report = List.first(body["reports"])++ date =+ case report["rows"] do+ [%{"dimensionValues" => [%{"value" => date_str}]}] ->+ Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()++ _ ->+ nil+ end++ {:ok, date}++ {:error, %{reason: %Finch.Response{body: body}}} ->+ Sentry.capture_message("Error fetching GA4 start date", extra: %{body: inspect(body)})+ {:error, body}++ {:error, %{reason: reason} = e} ->+ Sentry.capture_message("Error fetching GA4 start date", extra: %{error: inspect(e)})+ {:error, reason}+ end+ end++ defp reporting_api_url, do: "https://analyticsdata.googleapis.com"+ defp admin_api_url, do: "https://analyticsadmin.googleapis.com"+enddiff --git a/lib/plausible/google/ga4/report_request.ex b/lib/plausible/google/ga4/report_request.exnew file mode 100644index 0000000000..b859f17893--- /dev/null+++ b/lib/plausible/google/ga4/report_request.ex@@ -0,0 +1,122 @@+defmodule Plausible.Google.GA4.ReportRequest do+ @moduledoc """+ Report request struct for Google Analytics 4 API+ """++ defstruct [+ :dataset,+ :dimensions,+ :metrics,+ :date_range,+ :property,+ :access_token,+ :offset,+ :limit+ ]++ @type t() :: %__MODULE__{+ dataset: String.t(),+ dimensions: [String.t()],+ metrics: [String.t()],+ date_range: Date.Range.t(),+ property: term(),+ access_token: String.t(),+ offset: non_neg_integer(),+ limit: non_neg_integer()+ }++ def full_report do+ [+ %__MODULE__{+ dataset: "imported_visitors",+ dimensions: ["date"],+ metrics: [+ "totalUsers",+ "screenPageViews",+ "bounces = sessions - engagedSessions",+ "sessions",+ "userEngagementDuration"+ ]+ },+ %__MODULE__{+ dataset: "imported_sources",+ dimensions: [+ "date",+ "sessionSource",+ "sessionMedium",+ "sessionCampaignName",+ "sessionManualAdContent",+ "sessionGoogleAdsKeyword"+ ],+ metrics: [+ "totalUsers",+ "sessions",+ "bounces = sessions - engagedSessions",+ "userEngagementDuration"+ ]+ },+ %__MODULE__{+ dataset: "imported_pages",+ dimensions: ["date", "hostName", "pagePath"],+ # NOTE: no exits as GA4 DATA API does not provide that metric+ metrics: ["totalUsers", "screenPageViews", "userEngagementDuration"]+ },+ %__MODULE__{+ dataset: "imported_entry_pages",+ dimensions: ["date", "landingPage"],+ metrics: [+ "totalUsers",+ "sessions",+ "userEngagementDuration",+ "bounces = sessions - engagedSessions"+ ]+ },+ # NOTE: Skipping for now as there's no dimension directly mapping to exit page path+ # %__MODULE__{+ # dataset: "imported_exit_pages",+ # dimensions: ["date", "ga:exitPagePath"],+ # metrics: ["totalUsers", "sessions"]+ # },+ %__MODULE__{+ dataset: "imported_locations",+ dimensions: ["date", "countryId", "region", "city"],+ metrics: [+ "totalUsers",+ "sessions",+ "bounces = sessions - engagedSessions",+ "userEngagementDuration"+ ]+ },+ %__MODULE__{+ dataset: "imported_devices",+ dimensions: ["date", "deviceCategory"],+ metrics: [+ "totalUsers",+ "sessions",+ "bounces = sessions - engagedSessions",+ "userEngagementDuration"+ ]+ },+ %__MODULE__{+ dataset: "imported_browsers",+ dimensions: ["date", "browser"],+ metrics: [+ "totalUsers",+ "sessions",+ "bounces = sessions - engagedSessions",+ "userEngagementDuration"+ ]+ },+ %__MODULE__{+ dataset: "imported_operating_systems",+ dimensions: ["date", "operatingSystem"],+ metrics: [+ "totalUsers",+ "sessions",+ "bounces = sessions - engagedSessions",+ "userEngagementDuration"+ ]+ }+ ]+ end+enddiff --git a/lib/plausible/google/http.ex b/lib/plausible/google/http.exindex 93dad5910f..be55fbcf5e 100644--- a/lib/plausible/google/http.ex+++ b/lib/plausible/google/http.ex@@ -2,97 +2,6 @@ defmodule Plausible.Google.HTTP dorequire Loggeralias Plausible.HTTPClient- @spec get_report(Plausible.Google.ReportRequest.t()) ::- {:ok, {[map()], String.t() | nil}} | {:error, any()}- def get_report(%Plausible.Google.ReportRequest{} = report_request) do- params = %{- reportRequests: [- %{- viewId: report_request.view_id,- dateRanges: [- %{- startDate: report_request.date_range.first,- endDate: report_request.date_range.last- }- ],- dimensions: Enum.map(report_request.dimensions, &%{name: &1, histogramBuckets: []}),- metrics: Enum.map(report_request.metrics, &%{expression: &1}),- hideTotals: true,- hideValueRanges: true,- orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}],- pageSize: report_request.page_size,- pageToken: report_request.page_token- }- ]- }-- response =- HTTPClient.impl().post(- "#{reporting_api_url()}/v4/reports:batchGet",- [{"Authorization", "Bearer #{report_request.access_token}"}],- params,- receive_timeout: 60_000- )-- with {:ok, %{body: body}} <- response,- {:ok, report} <- parse_report_from_response(body),- token <- Map.get(report, "nextPageToken"),- {:ok, report} <- convert_to_maps(report) do- {:ok, {report, token}}- else- {:error, %{reason: %{status: status, body: body}}} ->- Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})- {:error, :request_failed}-- {:error, _reason} ->- {:error, :request_failed}- end- end-- defp parse_report_from_response(body) do- with %{"reports" => [report | _]} <- body do- {:ok, report}- else- _ ->- Sentry.Context.set_extra_context(%{google_analytics_response: body})-- Logger.error(- "Google Analytics: Failed to find report in response. Reason: #{inspect(body)}"- )-- {:error, {:invalid_response, body}}- end- end-- defp convert_to_maps(%{- "data" => %{} = data,- "columnHeader" => %{- "dimensions" => dimension_headers,- "metricHeader" => %{"metricHeaderEntries" => metric_headers}- }- }) do- metric_headers = Enum.map(metric_headers, & &1["name"])- rows = Map.get(data, "rows", [])-- report =- Enum.map(rows, fn %{"dimensions" => dimensions, "metrics" => [%{"values" => metrics}]} ->- metrics = Enum.zip(metric_headers, metrics)- dimensions = Enum.zip(dimension_headers, dimensions)- %{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}- end)-- {:ok, report}- end-- defp convert_to_maps(response) do- Logger.error(- "Google Analytics: Failed to read report in response. Reason: #{inspect(response)}"- )-- Sentry.Context.set_extra_context(%{google_analytics_response: response})- {:error, {:invalid_response, response}}- end-def list_sites(access_token) dourl = "#{api_url()}/webmasters/v3/sites"headers = [{"Content-Type", "application/json"}, {"Authorization", "Bearer #{access_token}"}]@@ -113,7 +22,7 @@ defmodule Plausible.Google.HTTP doendend- def fetch_access_token(code) do+ def fetch_access_token!(code) dourl = "#{api_url()}/oauth2/v4/token"headers = [{"Content-Type", "application/x-www-form-urlencoded"}]@@ -130,24 +39,6 @@ defmodule Plausible.Google.HTTP doresponse.bodyend- def list_views_for_user(access_token) do- url = "#{api_url()}/analytics/v3/management/accounts/~all/webproperties/~all/profiles"-- headers = [{"Authorization", "Bearer #{access_token}"}]-- case HTTPClient.impl().get(url, headers) do- {:ok, %Finch.Response{body: body, status: 200}} ->- {:ok, body}-- {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->- {:error, :authentication_failed}-- {:error, %HTTPClient.Non200Error{} = error} ->- Sentry.capture_message("Error listing GA views for user", extra: %{error: error})- {:error, :unknown}- end- end-def list_stats(access_token, property, date_range, limit, page \\ nil) doproperty = URI.encode_www_form(property)@@ -215,57 +106,9 @@ defmodule Plausible.Google.HTTP doendend- @earliest_valid_date "2005-01-01"- def get_analytics_start_date(view_id, access_token) do- params = %{- reportRequests: [- %{- viewId: view_id,- dateRanges: [- %{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}- ],- dimensions: [%{name: "ga:date", histogramBuckets: []}],- metrics: [%{expression: "ga:pageviews"}],- hideTotals: true,- hideValueRanges: true,- orderBys: [%{fieldName: "ga:date", sortOrder: "ASCENDING"}],- pageSize: 1- }- ]- }-- url = "#{reporting_api_url()}/v4/reports:batchGet"- headers = [{"Authorization", "Bearer #{access_token}"}]-- case HTTPClient.post(url, headers, params) do- {:ok, %Finch.Response{body: body, status: 200}} ->- report = List.first(body["reports"])-- date =- case report["data"]["rows"] do- [%{"dimensions" => [date_str]}] ->- Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()-- _ ->- nil- end-- {:ok, date}-- {:error, %{reason: %Finch.Response{body: body}}} ->- Sentry.capture_message("Error fetching Google view ID", extra: %{body: inspect(body)})- {:error, body}-- {:error, %{reason: reason} = e} ->- Sentry.capture_message("Error fetching Google view ID", extra: %{error: inspect(e)})- {:error, reason}- end- end-defp config, do: Application.get_env(:plausible, :google)defp client_id, do: Keyword.fetch!(config(), :client_id)defp client_secret, do: Keyword.fetch!(config(), :client_secret)- defp reporting_api_url, do: Keyword.fetch!(config(), :reporting_api_url)defp api_url, do: Keyword.fetch!(config(), :api_url)defp redirect_uri, do: PlausibleWeb.Endpoint.url() <> "/auth/google/callback"enddiff --git a/lib/plausible/google/ua/api.ex b/lib/plausible/google/ua/api.exnew file mode 100644index 0000000000..47d2cebebf--- /dev/null+++ b/lib/plausible/google/ua/api.ex@@ -0,0 +1,146 @@+defmodule Plausible.Google.UA.API do+ @moduledoc """+ API for Universal Analytics+ """++ alias Plausible.Google+ alias Plausible.Google.UA++ @type google_analytics_view() :: {view_name :: String.t(), view_id :: String.t()}++ @type import_auth :: {+ access_token :: String.t(),+ refresh_token :: String.t(),+ expires_at :: String.t()+ }++ @per_page 7_500+ @backoff_factor :timer.seconds(10)+ @max_attempts 5++ @spec list_views(access_token :: String.t()) ::+ {:ok, %{(hostname :: String.t()) => [google_analytics_view()]}} | {:error, term()}+ @doc """+ Lists Google Analytics views grouped by hostname.+ """+ def list_views(access_token) do+ case UA.HTTP.list_views_for_user(access_token) do+ {:ok, %{"items" => views}} ->+ views = Enum.group_by(views, &view_hostname/1, &view_names/1)+ {:ok, views}++ error ->+ error+ end+ end++ @spec get_view(access_token :: String.t(), lookup_id :: String.t()) ::+ {:ok, google_analytics_view()} | {:ok, nil} | {:error, term()}+ @doc """+ Returns a single Google Analytics view if the user has access to it.+ """+ def get_view(access_token, lookup_id) do+ case list_views(access_token) do+ {:ok, views} ->+ view =+ views+ |> Map.values()+ |> List.flatten()+ |> Enum.find(fn {_name, id} -> id == lookup_id end)++ {:ok, view}++ {:error, cause} ->+ {:error, cause}+ end+ end++ def get_analytics_start_date(access_token, view_id) do+ UA.HTTP.get_analytics_start_date(access_token, view_id)+ end++ @spec import_analytics(Date.Range.t(), String.t(), import_auth(), (String.t(), [map()] -> :ok)) ::+ :ok | {:error, term()}+ @doc """+ Imports stats from a Google Analytics UA view to a Plausible site.++ This function fetches Google Analytics reports in batches of #{@per_page} per+ request. The batches are then passed to persist callback.++ Requests to Google Analytics can fail, and are retried at most+ #{@max_attempts} times with an exponential backoff. Returns `:ok` when+ importing has finished or `{:error, term()}` when a request to GA failed too+ many times.++ Useful links:++ - [Feature documentation](https://plausible.io/docs/google-analytics-import)+ - [GA API reference](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest)+ - [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer)++ """+ def import_analytics(date_range, view_id, auth, persist_fn) do+ with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do+ do_import_analytics(date_range, view_id, access_token, persist_fn)+ end+ end++ defp do_import_analytics(date_range, view_id, access_token, persist_fn) do+ Enum.reduce_while(UA.ReportRequest.full_report(), :ok, fn report_request, :ok ->+ report_request = %UA.ReportRequest{+ report_request+ | date_range: date_range,+ view_id: view_id,+ access_token: access_token,+ page_token: nil,+ page_size: @per_page+ }++ case fetch_and_persist(report_request, persist_fn: persist_fn) do+ :ok -> {:cont, :ok}+ {:error, _} = error -> {:halt, error}+ end+ end)+ end++ @spec fetch_and_persist(UA.ReportRequest.t(), Keyword.t()) ::+ :ok | {:error, term()}+ def fetch_and_persist(%UA.ReportRequest{} = report_request, opts \\ []) do+ persist_fn = Keyword.fetch!(opts, :persist_fn)+ attempt = Keyword.get(opts, :attempt, 1)+ sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)++ case UA.HTTP.get_report(report_request) do+ {:ok, {rows, next_page_token}} ->+ :ok = persist_fn.(report_request.dataset, rows)++ if next_page_token do+ fetch_and_persist(+ %UA.ReportRequest{report_request | page_token: next_page_token},+ opts+ )+ else+ :ok+ end++ {:error, cause} ->+ if attempt >= @max_attempts do+ {:error, cause}+ else+ Process.sleep(attempt * sleep_time)+ fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))+ end+ end+ end++ defp view_hostname(view) do+ case view do+ %{"websiteUrl" => url} when is_binary(url) -> url |> URI.parse() |> Map.get(:host)+ _any -> "Others"+ end+ end++ defp view_names(%{"name" => name, "id" => id}) do+ {"#{id} - #{name}", id}+ end+enddiff --git a/lib/plausible/google/ua/http.ex b/lib/plausible/google/ua/http.exnew file mode 100644index 0000000000..f00712f5ec--- /dev/null+++ b/lib/plausible/google/ua/http.ex@@ -0,0 +1,167 @@+defmodule Plausible.Google.UA.HTTP do+ @moduledoc """+ HTTP client implementation for Universal Analytics API.+ """++ require Logger+ alias Plausible.HTTPClient++ @spec get_report(Plausible.Google.UA.ReportRequest.t()) ::+ {:ok, {[map()], String.t() | nil}} | {:error, any()}+ def get_report(%Plausible.Google.UA.ReportRequest{} = report_request) do+ params = %{+ reportRequests: [+ %{+ viewId: report_request.view_id,+ dateRanges: [+ %{+ startDate: report_request.date_range.first,+ endDate: report_request.date_range.last+ }+ ],+ dimensions: Enum.map(report_request.dimensions, &%{name: &1, histogramBuckets: []}),+ metrics: Enum.map(report_request.metrics, &%{expression: &1}),+ hideTotals: true,+ hideValueRanges: true,+ orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}],+ pageSize: report_request.page_size,+ pageToken: report_request.page_token+ }+ ]+ }++ response =+ HTTPClient.impl().post(+ "#{reporting_api_url()}/v4/reports:batchGet",+ [{"Authorization", "Bearer #{report_request.access_token}"}],+ params,+ receive_timeout: 60_000+ )++ with {:ok, %{body: body}} <- response,+ {:ok, report} <- parse_report_from_response(body),+ token <- Map.get(report, "nextPageToken"),+ {:ok, report} <- convert_to_maps(report) do+ {:ok, {report, token}}+ else+ {:error, %{reason: %{status: status, body: body}}} ->+ Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})+ {:error, :request_failed}++ {:error, _reason} ->+ {:error, :request_failed}+ end+ end++ defp parse_report_from_response(%{"reports" => [report | _]}) do+ {:ok, report}+ end++ defp parse_report_from_response(body) do+ Sentry.Context.set_extra_context(%{universal_analytics_response: body})++ Logger.error(+ "Universal Analytics: Failed to find report in response. Reason: #{inspect(body)}"+ )++ {:error, {:invalid_response, body}}+ end++ defp convert_to_maps(%{+ "data" => %{} = data,+ "columnHeader" => %{+ "dimensions" => dimension_headers,+ "metricHeader" => %{"metricHeaderEntries" => metric_headers}+ }+ }) do+ metric_headers = Enum.map(metric_headers, & &1["name"])+ rows = Map.get(data, "rows", [])++ report =+ Enum.map(rows, fn %{"dimensions" => dimensions, "metrics" => [%{"values" => metrics}]} ->+ metrics = Enum.zip(metric_headers, metrics)+ dimensions = Enum.zip(dimension_headers, dimensions)+ %{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}+ end)++ {:ok, report}+ end++ defp convert_to_maps(response) do+ Logger.error(+ "Universal Analytics: Failed to read report in response. Reason: #{inspect(response)}"+ )++ Sentry.Context.set_extra_context(%{universal_analytics_response: response})+ {:error, {:invalid_response, response}}+ end++ def list_views_for_user(access_token) do+ url = "#{api_url()}/analytics/v3/management/accounts/~all/webproperties/~all/profiles"++ headers = [{"Authorization", "Bearer #{access_token}"}]++ case HTTPClient.impl().get(url, headers) do+ {:ok, %Finch.Response{body: body, status: 200}} ->+ {:ok, body}++ {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->+ {:error, :authentication_failed}++ {:error, %HTTPClient.Non200Error{} = error} ->+ Sentry.capture_message("Error listing GA views for user", extra: %{error: error})+ {:error, :unknown}+ end+ end++ @earliest_valid_date "2005-01-01"+ def get_analytics_start_date(access_token, view_id) do+ params = %{+ reportRequests: [+ %{+ viewId: view_id,+ dateRanges: [+ %{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Timex.today())}+ ],+ dimensions: [%{name: "ga:date", histogramBuckets: []}],+ metrics: [%{expression: "ga:pageviews"}],+ hideTotals: true,+ hideValueRanges: true,+ orderBys: [%{fieldName: "ga:date", sortOrder: "ASCENDING"}],+ pageSize: 1+ }+ ]+ }++ url = "#{reporting_api_url()}/v4/reports:batchGet"+ headers = [{"Authorization", "Bearer #{access_token}"}]++ case HTTPClient.post(url, headers, params) do+ {:ok, %Finch.Response{body: body, status: 200}} ->+ report = List.first(body["reports"])++ date =+ case report["data"]["rows"] do+ [%{"dimensions" => [date_str]}] ->+ Timex.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()++ _ ->+ nil+ end++ {:ok, date}++ {:error, %{reason: %Finch.Response{body: body}}} ->+ Sentry.capture_message("Error fetching UA start date", extra: %{body: inspect(body)})+ {:error, body}++ {:error, %{reason: reason} = e} ->+ Sentry.capture_message("Error fetching UA start date", extra: %{error: inspect(e)})+ {:error, reason}+ end+ end++ defp config, do: Application.get_env(:plausible, :google)+ defp reporting_api_url, do: Keyword.fetch!(config(), :reporting_api_url)+ defp api_url, do: Keyword.fetch!(config(), :api_url)+enddiff --git a/lib/plausible/google/report_request.ex b/lib/plausible/google/ua/report_request.exsimilarity index 95%rename from lib/plausible/google/report_request.exrename to lib/plausible/google/ua/report_request.exindex fcc8fe2c12..e55419dd7d 100644--- a/lib/plausible/google/report_request.ex+++ b/lib/plausible/google/ua/report_request.ex@@ -1,4 +1,8 @@-defmodule Plausible.Google.ReportRequest do+defmodule Plausible.Google.UA.ReportRequest do+ @moduledoc """+ Report request struct for Universal Analytics API+ """+defstruct [:dataset,:dimensions,diff --git a/lib/plausible/imported/buffer.ex b/lib/plausible/imported/buffer.exindex 554cfaf701..fccbb62784 100644--- a/lib/plausible/imported/buffer.ex+++ b/lib/plausible/imported/buffer.ex@@ -8,12 +8,13 @@ defmodule Plausible.Imported.Buffer douse GenServerrequire Logger- def start_link do- GenServer.start_link(__MODULE__, nil)+ def start_link(opts \\ []) do+ GenServer.start_link(__MODULE__, opts)end- def init(_opts) do- {:ok, %{buffers: %{}}}+ def init(opts) do+ flush_interval = Keyword.get(opts, :flush_interval_ms, 1000)+ {:ok, %{flush_interval: flush_interval, buffers: %{}}}end@spec insert_many(pid(), term(), [map()]) :: :ok@@ -68,14 +69,14 @@ defmodule Plausible.Imported.Buffer dodef handle_call(:flush_all_buffers, _from, state) doEnum.each(state.buffers, fn {table_name, records} ->- flush_buffer(records, table_name)+ flush_buffer(records, table_name, state.flush_interval)end){:reply, :ok, put_in(state.buffers, %{})}enddef handle_continue({:flush, table_name}, state) do- flush_buffer(state.buffers[table_name], table_name)+ flush_buffer(state.buffers[table_name], table_name, state.flush_interval){:noreply, put_in(state.buffers[table_name], [])}end@@ -85,10 +86,10 @@ defmodule Plausible.Imported.Buffer do|> Keyword.fetch!(:max_buffer_size)end- defp flush_buffer(records, table_name) do+ defp flush_buffer(records, table_name, flush_interval) do# Clickhouse does not recommend sending more than 1 INSERT operation per second, and this# sleep call slows down the flushing- Process.sleep(1000)+ Process.sleep(flush_interval)Logger.info("Import: Flushing #{length(records)} from #{table_name} buffer")insert_all(table_name, records)diff --git a/lib/plausible/imported/google_analytics4.ex b/lib/plausible/imported/google_analytics4.exnew file mode 100644index 0000000000..5832421334--- /dev/null+++ b/lib/plausible/imported/google_analytics4.ex@@ -0,0 +1,253 @@+defmodule Plausible.Imported.GoogleAnalytics4 do+ @moduledoc """+ Import implementation for Google Analytics 4.+ """++ use Plausible.Imported.Importer++ @missing_values ["(none)", "(not set)", "(not provided)", "(other)"]++ @impl true+ def name(), do: :google_analytics_4++ @impl true+ def label(), do: "Google Analytics 4"++ @impl true+ def email_template(), do: "google_analytics_import.html"++ @impl true+ def parse_args(+ %{"property" => property, "start_date" => start_date, "end_date" => end_date} = args+ ) do+ start_date = Date.from_iso8601!(start_date)+ end_date = Date.from_iso8601!(end_date)+ date_range = Date.range(start_date, end_date)++ auth = {+ Map.fetch!(args, "access_token"),+ Map.fetch!(args, "refresh_token"),+ Map.fetch!(args, "token_expires_at")+ }++ [+ property: property,+ date_range: date_range,+ auth: auth+ ]+ end++ @doc """+ Imports stats from a Google Analytics 4 property to a Plausible site.++ This function fetches Google Analytics 4 reports which are then passed in batches+ to Clickhouse by the `Plausible.Imported.Buffer` process.+ """+ @impl true+ def import_data(site_import, opts) do+ date_range = Keyword.fetch!(opts, :date_range)+ property = Keyword.fetch!(opts, :property)+ auth = Keyword.fetch!(opts, :auth)+ flush_interval_ms = Keyword.get(opts, :flush_interval_ms, 1000)++ {:ok, buffer} = Plausible.Imported.Buffer.start_link(flush_interval_ms: flush_interval_ms)++ persist_fn = fn table, rows ->+ records = from_report(rows, site_import.site_id, site_import.id, table)+ Plausible.Imported.Buffer.insert_many(buffer, table, records)+ end++ try do+ Plausible.Google.GA4.API.import_analytics(date_range, property, auth, persist_fn)+ after+ Plausible.Imported.Buffer.flush(buffer)+ Plausible.Imported.Buffer.stop(buffer)+ end+ end++ def from_report(nil, _site_id, _import_id, _metric), do: nil++ def from_report(data, site_id, import_id, table) do+ Enum.reduce(data, [], fn row, acc ->+ if Map.get(row.dimensions, "date") in @missing_values do+ acc+ else+ [new_from_report(site_id, import_id, table, row) | acc]+ end+ end)+ end++ defp parse_number(nr) do+ {float, ""} = Float.parse(nr)+ round(float)+ end++ defp new_from_report(site_id, import_id, "imported_visitors", row) do+ %{+ site_id: site_id,+ import_id: import_id,+ date: get_date(row),+ visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),+ pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),+ bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),+ visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),+ visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()+ }+ end++ defp new_from_report(site_id, import_id, "imported_sources", row) do+ %{+ site_id: site_id,+ import_id: import_id,+ date: get_date(row),+ source: row.dimensions |> Map.fetch!("sessionSource") |> parse_referrer(),+ utm_medium: row.dimensions |> Map.fetch!("sessionMedium") |> default_if_missing(),+ utm_campaign: row.dimensions |> Map.fetch!("sessionCampaignName") |> default_if_missing(),+ utm_content: row.dimensions |> Map.fetch!("sessionManualAdContent") |> default_if_missing(),+ utm_term: row.dimensions |> Map.fetch!("sessionGoogleAdsKeyword") |> default_if_missing(),+ visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),+ visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),+ bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),+ visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()+ }+ end++ defp new_from_report(site_id, import_id, "imported_pages", row) do+ %{+ site_id: site_id,+ import_id: import_id,+ date: get_date(row),+ hostname: row.dimensions |> Map.fetch!("hostName") |> String.replace_prefix("www.", ""),+ page: row.dimensions |> Map.fetch!("pagePath") |> URI.parse() |> Map.get(:path),+ visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),+ pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),+ # NOTE: no exits metric in GA4 API currently+ exits: 0,+ time_on_page: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()+ }+ end++ defp new_from_report(site_id, import_id, "imported_entry_pages", row) do+ %{+ site_id: site_id,+ import_id: import_id,+ date: get_date(row),+ entry_page: row.dimensions |> Map.fetch!("landingPage"),+ visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),+ entrances: row.metrics |> Map.fetch!("sessions") |> parse_number(),+ visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),+ bounces: row.metrics |> Map.fetch!("bounces") |> parse_number()+ }+ end++ # NOTE: no exit pages metrics in GA4 API available for now+ # defp new_from_report(site_id, import_id, "imported_exit_pages", row) do+ # %{+ # site_id: site_id,+ # import_id: import_id,+ # date: get_date(row),+ # exit_page: Map.fetch!(row.dimensions, "exitPage"),+ # visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),+ # exits: row.metrics |> Map.fetch!("sessions") |> parse_number()+ # }+ # end++ defp new_from_report(site_id, import_id, "imported_locations", row) do+ country_code = row.dimensions |> Map.fetch!("countryId") |> default_if_missing("")+ city_name = row.dimensions |> Map.fetch!("city") |> default_if_missing("")+ city_data = Location.get_city(city_name, country_code)++ %{+ site_id: site_id,+ import_id: import_id,+ date: get_date(row),+ country: country_code,+ region: row.dimensions |> Map.fetch!("region") |> default_if_missing(""),+ city: city_data && city_data.id,+ visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),+ visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),+ bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),+ visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()+ }+ end++ defp new_from_report(site_id, import_id, "imported_devices", row) do+ %{+ site_id: site_id,+ import_id: import_id,+ date: get_date(row),+ device: row.dimensions |> Map.fetch!("deviceCategory") |> String.capitalize(),+ visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),+ visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),+ bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),+ visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()+ }+ end++ @browser_google_to_plausible %{+ "User-Agent:Opera" => "Opera",+ "Mozilla Compatible Agent" => "Mobile App",+ "Android Webview" => "Mobile App",+ "Android Browser" => "Mobile App",+ "Safari (in-app)" => "Mobile App",+ "User-Agent: Mozilla" => "Firefox",+ "(not set)" => ""+ }++ defp new_from_report(site_id, import_id, "imported_browsers", row) do+ browser = Map.fetch!(row.dimensions, "browser")++ %{+ site_id: site_id,+ import_id: import_id,+ date: get_date(row),+ browser: Map.get(@browser_google_to_plausible, browser, browser),+ visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),+ visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),+ bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),+ visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()+ }+ end++ @os_google_to_plausible %{+ "Macintosh" => "Mac",+ "Linux" => "GNU/Linux",+ "(not set)" => ""+ }++ defp new_from_report(site_id, import_id, "imported_operating_systems", row) do+ os = Map.fetch!(row.dimensions, "operatingSystem")++ %{+ site_id: site_id,+ import_id: import_id,+ date: get_date(row),+ operating_system: Map.get(@os_google_to_plausible, os, os),+ visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),+ visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),+ bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),+ visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()+ }+ end++ defp get_date(%{dimensions: %{"date" => date}}) do+ date+ |> Timex.parse!("%Y%m%d", :strftime)+ |> NaiveDateTime.to_date()+ end++ defp default_if_missing(value, default \\ nil)+ defp default_if_missing(value, default) when value in @missing_values, do: default+ defp default_if_missing(value, _default), do: value++ defp parse_referrer(nil), do: nil+ defp parse_referrer("(direct)"), do: nil+ defp parse_referrer("google"), do: "Google"+ defp parse_referrer("bing"), do: "Bing"+ defp parse_referrer("duckduckgo"), do: "DuckDuckGo"++ defp parse_referrer(ref) do+ RefInspector.parse("https://" <> ref)+ |> PlausibleWeb.RefInspector.parse()+ end+enddiff --git a/lib/plausible/imported/import_sources.ex b/lib/plausible/imported/import_sources.exindex bcf1eb8d45..96a149b19b 100644--- a/lib/plausible/imported/import_sources.ex+++ b/lib/plausible/imported/import_sources.ex@@ -4,6 +4,7 @@ defmodule Plausible.Imported.ImportSources do"""@sources [+ Plausible.Imported.GoogleAnalytics4,Plausible.Imported.UniversalAnalytics,Plausible.Imported.NoopImporter,Plausible.Imported.CSVImporterdiff --git a/lib/plausible/imported/universal_analytics.ex b/lib/plausible/imported/universal_analytics.exindex 1c7a1e093e..1d0dcb2e8a 100644--- a/lib/plausible/imported/universal_analytics.ex+++ b/lib/plausible/imported/universal_analytics.ex@@ -102,7 +102,7 @@ defmodule Plausible.Imported.UniversalAnalytics doendtry do- Plausible.Google.Api.import_analytics(date_range, view_id, auth, persist_fn)+ Plausible.Google.UA.API.import_analytics(date_range, view_id, auth, persist_fn)afterPlausible.Imported.Buffer.flush(buffer)Plausible.Imported.Buffer.stop(buffer)diff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.exindex 11a68efc52..a242413710 100644--- a/lib/plausible_web/controllers/auth_controller.ex+++ b/lib/plausible_web/controllers/auth_controller.ex@@ -697,13 +697,16 @@ defmodule PlausibleWeb.AuthController doenddef google_auth_callback(conn, %{"error" => error, "state" => state} = params) do- [site_id, _redirected_to, legacy] =+ [site_id, _redirected_to, legacy, _ga4] =case Jason.decode!(state) do[site_id, redirect_to] ->- [site_id, redirect_to, true]+ [site_id, redirect_to, true, false][site_id, redirect_to, legacy] ->- [site_id, redirect_to, legacy]+ [site_id, redirect_to, legacy, false]++ [site_id, redirect_to, legacy, ga4] ->+ [site_id, redirect_to, legacy, ga4]endsite = Repo.get(Plausible.Site, site_id)@@ -745,15 +748,18 @@ defmodule PlausibleWeb.AuthController doenddef google_auth_callback(conn, %{"code" => code, "state" => state}) do- res = Plausible.Google.HTTP.fetch_access_token(code)+ res = Plausible.Google.API.fetch_access_token!(code)- [site_id, redirect_to, legacy] =+ [site_id, redirect_to, legacy, ga4] =case Jason.decode!(state) do[site_id, redirect_to] ->- [site_id, redirect_to, true]+ [site_id, redirect_to, true, false][site_id, redirect_to, legacy] ->- [site_id, redirect_to, legacy]+ [site_id, redirect_to, legacy, false]++ [site_id, redirect_to, legacy, ga4] ->+ [site_id, redirect_to, legacy, ga4]endsite = Repo.get(Plausible.Site, site_id)@@ -761,15 +767,26 @@ defmodule PlausibleWeb.AuthController docase redirect_to do"import" ->- redirect(conn,- external:- Routes.site_path(conn, :import_from_google_view_id_form, site.domain,- access_token: res["access_token"],- refresh_token: res["refresh_token"],- expires_at: NaiveDateTime.to_iso8601(expires_at),- legacy: legacy- )- )+ if ga4 do+ redirect(conn,+ external:+ Routes.google_analytics4_path(conn, :property_form, site.domain,+ access_token: res["access_token"],+ refresh_token: res["refresh_token"],+ expires_at: NaiveDateTime.to_iso8601(expires_at)+ )+ )+ else+ redirect(conn,+ external:+ Routes.universal_analytics_path(conn, :view_id_form, site.domain,+ access_token: res["access_token"],+ refresh_token: res["refresh_token"],+ expires_at: NaiveDateTime.to_iso8601(expires_at),+ legacy: legacy+ )+ )+ end_ ->id_token = res["id_token"]diff --git a/lib/plausible_web/controllers/google_analytics4_controller.ex b/lib/plausible_web/controllers/google_analytics4_controller.exnew file mode 100644index 0000000000..3fd55d3ab6--- /dev/null+++ b/lib/plausible_web/controllers/google_analytics4_controller.ex@@ -0,0 +1,143 @@+defmodule PlausibleWeb.GoogleAnalytics4Controller do+ use PlausibleWeb, :controller++ plug(PlausibleWeb.RequireAccountPlug)++ plug(PlausibleWeb.AuthorizeSiteAccess, [:owner, :admin, :super_admin])++ def property_form(conn, %{+ "access_token" => access_token,+ "refresh_token" => refresh_token,+ "expires_at" => expires_at+ }) do+ redirect_route = Routes.site_path(conn, :settings_imports_exports, conn.assigns.site.domain)++ case Plausible.Google.GA4.API.list_properties(access_token) do+ {:ok, properties} ->+ conn+ |> assign(:skip_plausible_tracking, true)+ |> render("property_form.html",+ access_token: access_token,+ refresh_token: refresh_token,+ expires_at: expires_at,+ site: conn.assigns.site,+ properties: properties,+ layout: {PlausibleWeb.LayoutView, "focus.html"}+ )++ {:error, :authentication_failed} ->+ conn+ |> put_flash(+ :error,+ "We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again."+ )+ |> redirect(external: redirect_route)++ {:error, _any} ->+ conn+ |> put_flash(+ :error,+ "We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance."+ )+ |> redirect(external: redirect_route)+ end+ end++ def property(conn, %{+ "property" => property,+ "access_token" => access_token,+ "refresh_token" => refresh_token,+ "expires_at" => expires_at+ }) do+ site = conn.assigns.site+ start_date = Plausible.Google.GA4.API.get_analytics_start_date(access_token, property)++ case start_date do+ {:ok, nil} ->+ {:ok, properties} = Plausible.Google.GA4.API.list_properties(access_token)++ conn+ |> assign(:skip_plausible_tracking, true)+ |> render("property_form.html",+ access_token: access_token,+ refresh_token: refresh_token,+ expires_at: expires_at,+ site: site,+ properties: properties,+ selected_property_error: "No data found. Nothing to import",+ layout: {PlausibleWeb.LayoutView, "focus.html"}+ )++ {:ok, _date} ->+ redirect(conn,+ to:+ Routes.google_analytics4_path(conn, :confirm, site.domain,+ property: property,+ access_token: access_token,+ refresh_token: refresh_token,+ expires_at: expires_at+ )+ )+ end+ end++ def confirm(conn, %{+ "property" => property,+ "access_token" => access_token,+ "refresh_token" => refresh_token,+ "expires_at" => expires_at+ }) do+ site = conn.assigns.site++ start_date = Plausible.Google.GA4.API.get_analytics_start_date(access_token, property)++ end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone)++ {:ok, {property_name, property}} =+ Plausible.Google.GA4.API.get_property(access_token, property)++ conn+ |> assign(:skip_plausible_tracking, true)+ |> render("confirm.html",+ access_token: access_token,+ refresh_token: refresh_token,+ expires_at: expires_at,+ site: site,+ selected_property: property,+ selected_property_name: property_name,+ start_date: start_date,+ end_date: end_date,+ layout: {PlausibleWeb.LayoutView, "focus.html"}+ )+ end++ def import(conn, %{+ "property" => property,+ "start_date" => start_date,+ "end_date" => end_date,+ "access_token" => access_token,+ "refresh_token" => refresh_token,+ "expires_at" => expires_at+ }) do+ site = conn.assigns.site+ current_user = conn.assigns.current_user++ redirect_route = Routes.site_path(conn, :settings_imports_exports, site.domain)++ {:ok, _} =+ Plausible.Imported.GoogleAnalytics4.new_import(+ site,+ current_user,+ property: property,+ start_date: start_date,+ end_date: end_date,+ access_token: access_token,+ refresh_token: refresh_token,+ token_expires_at: expires_at+ )++ conn+ |> put_flash(:success, "Import scheduled. An email will be sent when it completes.")+ |> redirect(external: redirect_route)+ end+enddiff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.exindex 51c04c7991..aa38531cc7 100644--- a/lib/plausible_web/controllers/site_controller.ex+++ b/lib/plausible_web/controllers/site_controller.ex@@ -232,7 +232,7 @@ defmodule PlausibleWeb.SiteController dosearch_console_domains =if site.google_auth do- Plausible.Google.Api.fetch_verified_properties(site.google_auth)+ Plausible.Google.API.fetch_verified_properties(site.google_auth)endimported_pageviews =@@ -641,198 +641,6 @@ defmodule PlausibleWeb.SiteController doendend- def import_from_google_user_metric_notice(conn, %{- "view_id" => view_id,- "access_token" => access_token,- "refresh_token" => refresh_token,- "expires_at" => expires_at,- "legacy" => legacy- }) do- site = conn.assigns[:site]-- conn- |> assign(:skip_plausible_tracking, true)- |> render("import_from_google_user_metric_form.html",- site: site,- view_id: view_id,- access_token: access_token,- refresh_token: refresh_token,- expires_at: expires_at,- legacy: legacy,- layout: {PlausibleWeb.LayoutView, "focus.html"}- )- end-- def import_from_google_view_id_form(conn, %{- "access_token" => access_token,- "refresh_token" => refresh_token,- "expires_at" => expires_at,- "legacy" => legacy- }) do- redirect_route =- if legacy == "true" do- Routes.site_path(conn, :settings_integrations, conn.assigns.site.domain)- else- Routes.site_path(conn, :settings_imports_exports, conn.assigns.site.domain)- end-- case Plausible.Google.Api.list_views(access_token) do- {:ok, view_ids} ->- conn- |> assign(:skip_plausible_tracking, true)- |> render("import_from_google_view_id_form.html",- access_token: access_token,- refresh_token: refresh_token,- expires_at: expires_at,- site: conn.assigns.site,- view_ids: view_ids,- legacy: legacy,- layout: {PlausibleWeb.LayoutView, "focus.html"}- )-- {:error, :authentication_failed} ->- conn- |> put_flash(- :error,- "We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again."- )- |> redirect(external: redirect_route)-- {:error, _any} ->- conn- |> put_flash(- :error,- "We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance."- )- |> redirect(external: redirect_route)- end- end-- # see https://stackoverflow.com/a/57416769- @google_analytics_new_user_metric_date ~D[2016-08-24]- def import_from_google_view_id(conn, %{- "view_id" => view_id,- "access_token" => access_token,- "refresh_token" => refresh_token,- "expires_at" => expires_at,- "legacy" => legacy- }) do- site = conn.assigns[:site]- start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token)-- case start_date do- {:ok, nil} ->- site = conn.assigns[:site]- {:ok, view_ids} = Plausible.Google.Api.list_views(access_token)-- conn- |> assign(:skip_plausible_tracking, true)- |> render("import_from_google_view_id_form.html",- access_token: access_token,- refresh_token: refresh_token,- expires_at: expires_at,- site: site,- view_ids: view_ids,- selected_view_id_error: "No data found. Nothing to import",- legacy: legacy,- layout: {PlausibleWeb.LayoutView, "focus.html"}- )-- {:ok, date} ->- if Timex.before?(date, @google_analytics_new_user_metric_date) do- redirect(conn,- to:- Routes.site_path(conn, :import_from_google_user_metric_notice, site.domain,- view_id: view_id,- access_token: access_token,- refresh_token: refresh_token,- expires_at: expires_at,- legacy: legacy- )- )- else- redirect(conn,- to:- Routes.site_path(conn, :import_from_google_confirm, site.domain,- view_id: view_id,- access_token: access_token,- refresh_token: refresh_token,- expires_at: expires_at,- legacy: legacy- )- )- end- end- end-- def import_from_google_confirm(conn, %{- "view_id" => view_id,- "access_token" => access_token,- "refresh_token" => refresh_token,- "expires_at" => expires_at,- "legacy" => legacy- }) do- site = conn.assigns[:site]-- start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token)-- end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone)-- {:ok, {view_name, view_id}} = Plausible.Google.Api.get_view(access_token, view_id)-- conn- |> assign(:skip_plausible_tracking, true)- |> render("import_from_google_confirm.html",- access_token: access_token,- refresh_token: refresh_token,- expires_at: expires_at,- site: site,- selected_view_id: view_id,- selected_view_id_name: view_name,- start_date: start_date,- end_date: end_date,- legacy: legacy,- layout: {PlausibleWeb.LayoutView, "focus.html"}- )- end-- def import_from_google(conn, %{- "view_id" => view_id,- "start_date" => start_date,- "end_date" => end_date,- "access_token" => access_token,- "refresh_token" => refresh_token,- "expires_at" => expires_at,- "legacy" => legacy- }) do- site = conn.assigns.site- current_user = conn.assigns.current_user-- redirect_route =- if legacy == "true" do- Routes.site_path(conn, :settings_integrations, site.domain)- else- Routes.site_path(conn, :settings_imports_exports, site.domain)- end-- {:ok, _} =- Plausible.Imported.UniversalAnalytics.new_import(- site,- current_user,- view_id: view_id,- start_date: start_date,- end_date: end_date,- access_token: access_token,- refresh_token: refresh_token,- token_expires_at: expires_at,- legacy: legacy == "true"- )-- conn- |> put_flash(:success, "Import scheduled. An email will be sent when it completes.")- |> redirect(external: redirect_route)- end-def forget_import(conn, %{"import_id" => import_id}) dosite = conn.assigns.sitediff --git a/lib/plausible_web/controllers/universal_analytics_controller.ex b/lib/plausible_web/controllers/universal_analytics_controller.exnew file mode 100644index 0000000000..16a4bea64e--- /dev/null+++ b/lib/plausible_web/controllers/universal_analytics_controller.ex@@ -0,0 +1,198 @@+defmodule PlausibleWeb.UniversalAnalyticsController do+ use PlausibleWeb, :controller++ plug(PlausibleWeb.RequireAccountPlug)++ plug(PlausibleWeb.AuthorizeSiteAccess, [:owner, :admin, :super_admin])++ def user_metric_notice(conn, %{+ "view_id" => view_id,+ "access_token" => access_token,+ "refresh_token" => refresh_token,+ "expires_at" => expires_at,+ "legacy" => legacy+ }) do+ site = conn.assigns.site++ conn+ |> assign(:skip_plausible_tracking, true)+ |> render("user_metric_form.html",+ site: site,+ view_id: view_id,+ access_token: access_token,+ refresh_token: refresh_token,+ expires_at: expires_at,+ legacy: legacy,+ layout: {PlausibleWeb.LayoutView, "focus.html"}+ )+ end++ def view_id_form(conn, %{+ "access_token" => access_token,+ "refresh_token" => refresh_token,+ "expires_at" => expires_at,+ "legacy" => legacy+ }) do+ redirect_route =+ if legacy == "true" do+ Routes.site_path(conn, :settings_integrations, conn.assigns.site.domain)+ else+ Routes.site_path(conn, :settings_imports_exports, conn.assigns.site.domain)+ end++ case Plausible.Google.UA.API.list_views(access_token) do+ {:ok, view_ids} ->+ conn+ |> assign(:skip_plausible_tracking, true)+ |> render("view_id_form.html",+ access_token: access_token,+ refresh_token: refresh_token,+ expires_at: expires_at,+ site: conn.assigns.site,+ view_ids: view_ids,+ legacy: legacy,+ layout: {PlausibleWeb.LayoutView, "focus.html"}+ )++ {:error, :authentication_failed} ->+ conn+ |> put_flash(+ :error,+ "We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again."+ )+ |> redirect(external: redirect_route)++ {:error, _any} ->+ conn+ |> put_flash(+ :error,+ "We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance."+ )+ |> redirect(external: redirect_route)+ end+ end++ # see https://stackoverflow.com/a/57416769+ @google_analytics_new_user_metric_date ~D[2016-08-24]+ def view_id(conn, %{+ "view_id" => view_id,+ "access_token" => access_token,+ "refresh_token" => refresh_token,+ "expires_at" => expires_at,+ "legacy" => legacy+ }) do+ site = conn.assigns.site+ start_date = Plausible.Google.UA.API.get_analytics_start_date(access_token, view_id)++ case start_date do+ {:ok, nil} ->+ {:ok, view_ids} = Plausible.Google.UA.API.list_views(access_token)++ conn+ |> assign(:skip_plausible_tracking, true)+ |> render("view_id_form.html",+ access_token: access_token,+ refresh_token: refresh_token,+ expires_at: expires_at,+ site: site,+ view_ids: view_ids,+ selected_view_id_error: "No data found. Nothing to import",+ legacy: legacy,+ layout: {PlausibleWeb.LayoutView, "focus.html"}+ )++ {:ok, date} ->+ if Timex.before?(date, @google_analytics_new_user_metric_date) do+ redirect(conn,+ to:+ Routes.universal_analytics_path(conn, :user_metric_notice, site.domain,+ view_id: view_id,+ access_token: access_token,+ refresh_token: refresh_token,+ expires_at: expires_at,+ legacy: legacy+ )+ )+ else+ redirect(conn,+ to:+ Routes.universal_analytics_path(conn, :confirm, site.domain,+ view_id: view_id,+ access_token: access_token,+ refresh_token: refresh_token,+ expires_at: expires_at,+ legacy: legacy+ )+ )+ end+ end+ end++ def confirm(conn, %{+ "view_id" => view_id,+ "access_token" => access_token,+ "refresh_token" => refresh_token,+ "expires_at" => expires_at,+ "legacy" => legacy+ }) do+ site = conn.assigns.site++ start_date = Plausible.Google.UA.API.get_analytics_start_date(access_token, view_id)++ end_date = Plausible.Sites.native_stats_start_date(site) || Timex.today(site.timezone)++ {:ok, {view_name, view_id}} = Plausible.Google.UA.API.get_view(access_token, view_id)++ conn+ |> assign(:skip_plausible_tracking, true)+ |> render("confirm.html",+ access_token: access_token,+ refresh_token: refresh_token,+ expires_at: expires_at,+ site: site,+ selected_view_id: view_id,+ selected_view_id_name: view_name,+ start_date: start_date,+ end_date: end_date,+ legacy: legacy,+ layout: {PlausibleWeb.LayoutView, "focus.html"}+ )+ end++ def import(conn, %{+ "view_id" => view_id,+ "start_date" => start_date,+ "end_date" => end_date,+ "access_token" => access_token,+ "refresh_token" => refresh_token,+ "expires_at" => expires_at,+ "legacy" => legacy+ }) do+ site = conn.assigns.site+ current_user = conn.assigns.current_user++ redirect_route =+ if legacy == "true" do+ Routes.site_path(conn, :settings_integrations, site.domain)+ else+ Routes.site_path(conn, :settings_imports_exports, site.domain)+ end++ {:ok, _} =+ Plausible.Imported.UniversalAnalytics.new_import(+ site,+ current_user,+ view_id: view_id,+ start_date: start_date,+ end_date: end_date,+ access_token: access_token,+ refresh_token: refresh_token,+ token_expires_at: expires_at,+ legacy: legacy == "true"+ )++ conn+ |> put_flash(:success, "Import scheduled. An email will be sent when it completes.")+ |> redirect(external: redirect_route)+ end+enddiff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.exindex d1c414ac76..162dee3cad 100644--- a/lib/plausible_web/router.ex+++ b/lib/plausible_web/router.ex@@ -374,17 +374,27 @@ defmodule PlausibleWeb.Router dodelete "/:website/stats", SiteController, :reset_statsget "/:website/import/google-analytics/view-id",- SiteController,- :import_from_google_view_id_form+ UniversalAnalyticsController,+ :view_id_form- post "/:website/import/google-analytics/view-id", SiteController, :import_from_google_view_id+ post "/:website/import/google-analytics/view-id", UniversalAnalyticsController, :view_idget "/:website/import/google-analytics/user-metric",- SiteController,- :import_from_google_user_metric_notice+ UniversalAnalyticsController,+ :user_metric_notice++ get "/:website/import/google-analytics/confirm", UniversalAnalyticsController, :confirm+ post "/:website/settings/google-import", UniversalAnalyticsController, :import++ get "/:website/import/google-analytics4/property",+ GoogleAnalytics4Controller,+ :property_form++ post "/:website/import/google-analytics4/property", GoogleAnalytics4Controller, :property++ get "/:website/import/google-analytics4/confirm", GoogleAnalytics4Controller, :confirm+ post "/:website/settings/google4-import", GoogleAnalytics4Controller, :import- get "/:website/import/google-analytics/confirm", SiteController, :import_from_google_confirm- post "/:website/settings/google-import", SiteController, :import_from_googledelete "/:website/settings/forget-imported", SiteController, :forget_importeddelete "/:website/settings/forget-import/:import_id", SiteController, :forget_importdiff --git a/lib/plausible_web/templates/google_analytics4/confirm.html.heex b/lib/plausible_web/templates/google_analytics4/confirm.html.heexnew file mode 100644index 0000000000..b0f63048ba--- /dev/null+++ b/lib/plausible_web/templates/google_analytics4/confirm.html.heex@@ -0,0 +1,46 @@+<%= form_for @conn, Routes.google_analytics4_path(@conn, :import, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>+ <h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>++ <%= hidden_input(f, :access_token, value: @access_token) %>+ <%= hidden_input(f, :refresh_token, value: @refresh_token) %>+ <%= hidden_input(f, :expires_at, value: @expires_at) %>++ <%= case @start_date do %>+ <% {:ok, start_date} -> %>+ <div class="mt-6 text-sm text-gray-500 dark:text-gray-200">+ Stats from this property and time period will be imported from your Google Analytics 4 account to your Plausible dashboard+ </div>++ <div class="mt-6">+ <%= styled_label(f, :property, "Google Analytics 4 property") %>+ <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">+ <%= @selected_property_name %>+ </span>+ <%= hidden_input(f, :property, readonly: "true", value: @selected_property) %>+ </div>+ <div class="flex justify-between mt-3">+ <div class="w-36">+ <%= styled_label(f, :start_date, "From") %>+ <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">+ <%= PlausibleWeb.EmailView.date_format(start_date) %>+ </span>+ <%= hidden_input(f, :start_date, value: start_date, readonly: "true") %>+ </div>+ <div class="align-middle pt-4 dark:text-gray-100">→</div>+ <div class="w-36">+ <%= styled_label(f, :end_date, "To") %>+ <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">+ <%= PlausibleWeb.EmailView.date_format(@end_date) %>+ </span>+ <%= hidden_input(f, :end_date, value: @end_date, readonly: "true") %>+ </div>+ </div>+ <% {:error, error} -> %>+ <p class="text-gray-700 dark:text-gray-300 mt-6">+ The following error occurred when fetching your Google Analytics 4 data.+ </p>+ <p class="text-red-700 font-medium mt-3"><%= error %></p>+ <% end %>++ <%= submit("Confirm import", class: "button mt-6") %>+<% end %>diff --git a/lib/plausible_web/templates/google_analytics4/property_form.html.heex b/lib/plausible_web/templates/google_analytics4/property_form.html.heexnew file mode 100644index 0000000000..c88461073c--- /dev/null+++ b/lib/plausible_web/templates/google_analytics4/property_form.html.heex@@ -0,0 +1,19 @@+<%= form_for @conn, Routes.google_analytics4_path(@conn, :property, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>+ <h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics 4</h2>++ <%= hidden_input(f, :access_token, value: @access_token) %>+ <%= hidden_input(f, :refresh_token, value: @refresh_token) %>+ <%= hidden_input(f, :expires_at, value: @expires_at) %>++ <div class="mt-6 text-sm text-gray-500 dark:text-gray-200">+ Choose the property in your Google Analytics 4 account that will be imported to the <%= @site.domain %> dashboard.+ </div>++ <div class="mt-3">+ <%= styled_label(f, :property, "Google Analytics 4 property") %>+ <%= styled_select(f, :property, @properties, prompt: "(Choose property)", required: "true") %>+ <%= styled_error(@conn.assigns[:selected_property_error]) %>+ </div>++ <%= submit("Continue ->", class: "button mt-6") %>+<% end %>diff --git a/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex b/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eexdeleted file mode 100644index e5792ed9f9..0000000000--- a/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex+++ /dev/null@@ -1,26 +0,0 @@-<div class="max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8">- <h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>-- <div class="mt-6 text-sm text-gray-500 dark:text-gray-200">- <p>- <svg xmlns="http://www.w3.org/2000/svg" class="h-4 w-4 inline text-orange-600" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">- <path stroke-linecap="round" stroke-linejoin="round" d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-3L13.732 4c-.77-1.333-2.694-1.333-3.464 0L3.34 16c-.77 1.333.192 3 1.732 3z" />- </svg>-- Important: Since your GA property includes data from before 23rd August 2016, you have to take an extra step to make sure we can import data smoothly.- </p>-- <ol class="mt-4">- <li>1. Navigate to the GA property you want to import from</li>- <li>2. Go to Admin > Property Settings > User Analysis</li>- <li>3. Make sure <i>Enable Users Metric in Reporting</i> is <b>OFF</b></li>- </ol>-- <p class="mt-4">- The setting may take a few minutes to take effect. If your imported data is showing 0 visitors in unexpected places, it's probably caused by this and you- can try importing again later.- </p>- </div>-- <%= link("Continue ->", to: Routes.site_path(@conn, :import_from_google_confirm, @site.domain, view_id: @view_id, access_token: @access_token, refresh_token: @refresh_token, expires_at: @expires_at, legacy: @legacy), class: "button mt-6") %>-</div>diff --git a/lib/plausible_web/templates/site/settings_google_import.html.heex b/lib/plausible_web/templates/site/settings_google_import.html.heexindex fb223f2982..c7d76b99c8 100644--- a/lib/plausible_web/templates/site/settings_google_import.html.heex+++ b/lib/plausible_web/templates/site/settings_google_import.html.heex@@ -92,7 +92,7 @@<% end %><PlausibleWeb.Components.Google.buttonid="analytics-connect"- to={Plausible.Google.Api.import_authorize_url(@site.id, "import", true)}+ to={Plausible.Google.API.import_authorize_url(@site.id, "import", legacy: true)}/><% end %><% else %>diff --git a/lib/plausible_web/templates/site/settings_imports_exports.html.heex b/lib/plausible_web/templates/site/settings_imports_exports.html.heexindex 9369143a17..b1b31f877c 100644--- a/lib/plausible_web/templates/site/settings_imports_exports.html.heex+++ b/lib/plausible_web/templates/site/settings_imports_exports.html.heex@@ -14,20 +14,22 @@<PlausibleWeb.Components.Generic.button_linkclass="w-36 h-20"theme="bright"- href={Plausible.Google.Api.import_authorize_url(@site.id, "import", false)}+ href={Plausible.Google.API.import_authorize_url(@site.id, "import", legacy: false)}><img src="/images/icon/universal_analytics_logo.svg" alt="New Universal Analytics import" /></PlausibleWeb.Components.Generic.button_link><PlausibleWeb.Components.Generic.button_link- class="w-36 h-20 opacity-40 cursor-not-allowed"+ class="w-36 h-20"theme="bright"- href=""+ href={+ Plausible.Google.API.import_authorize_url(@site.id, "import", legacy: false, ga4: true)+ }><imgsrc="/images/icon/google_analytics_4_logo.svg"width="110"- alt="New Universal Analytics import"+ alt="New Google Analytics 4 import"/></PlausibleWeb.Components.Generic.button_link>diff --git a/lib/plausible_web/templates/site/settings_search_console.html.heex b/lib/plausible_web/templates/site/settings_search_console.html.heexindex 5b5b6aff9c..c615fab3aa 100644--- a/lib/plausible_web/templates/site/settings_search_console.html.heex+++ b/lib/plausible_web/templates/site/settings_search_console.html.heex@@ -82,7 +82,7 @@<% else %><PlausibleWeb.Components.Google.buttonid="search-console-connect"- to={Plausible.Google.Api.search_console_authorize_url(@site.id, "search-console")}+ to={Plausible.Google.API.search_console_authorize_url(@site.id, "search-console")}/><div class="text-gray-700 dark:text-gray-300 mt-8">NB: You also need to set up your site ondiff --git a/lib/plausible_web/templates/site/import_from_google_confirm.html.eex b/lib/plausible_web/templates/universal_analytics/confirm.html.heexsimilarity index 52%rename from lib/plausible_web/templates/site/import_from_google_confirm.html.eexrename to lib/plausible_web/templates/universal_analytics/confirm.html.heexindex c46459aa94..3d6f143f32 100644--- a/lib/plausible_web/templates/site/import_from_google_confirm.html.eex+++ b/lib/plausible_web/templates/universal_analytics/confirm.html.heex@@ -1,4 +1,4 @@-<%= form_for @conn, Routes.site_path(@conn, :import_from_google, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>+<%= form_for @conn, Routes.universal_analytics_path(@conn, :import, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %><h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2><%= hidden_input(f, :access_token, value: @access_token) %>@@ -8,33 +8,40 @@<%= case @start_date do %><% {:ok, start_date} -> %>-<div class="mt-6 text-sm text-gray-500 dark:text-gray-200">Stats from this property and time period will be imported from your Google Analytics account to your Plausible dashboard</div><div class="mt-6"><%= styled_label(f, :view_id, "Google Analytics view") %>- <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800"><%= @selected_view_id_name %></span>- <%= hidden_input f, :view_id, readonly: "true", value: @selected_view_id %>+ <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">+ <%= @selected_view_id_name %>+ </span>+ <%= hidden_input(f, :view_id, readonly: "true", value: @selected_view_id) %></div><div class="flex justify-between mt-3"><div class="w-36">- <%= styled_label f, :start_date, "From" %>- <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800"><%= PlausibleWeb.EmailView.date_format(start_date) %></span>- <%= hidden_input f, :start_date, value: start_date, readonly: "true" %>+ <%= styled_label(f, :start_date, "From") %>+ <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">+ <%= PlausibleWeb.EmailView.date_format(start_date) %>+ </span>+ <%= hidden_input(f, :start_date, value: start_date, readonly: "true") %></div><div class="align-middle pt-4 dark:text-gray-100">→</div><div class="w-36">- <%= styled_label f, :end_date, "To" %>- <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800"><%= PlausibleWeb.EmailView.date_format(@end_date) %></span>- <%= hidden_input f, :end_date, value: @end_date, readonly: "true" %>+ <%= styled_label(f, :end_date, "To") %>+ <span class="block w-full text-base dark:text-gray-100 sm:text-sm dark:bg-gray-800">+ <%= PlausibleWeb.EmailView.date_format(@end_date) %>+ </span>+ <%= hidden_input(f, :end_date, value: @end_date, readonly: "true") %></div></div><% {:error, error} -> %>- <p class="text-gray-700 dark:text-gray-300 mt-6">The following error occurred when fetching your Google Analytics data.</p>+ <p class="text-gray-700 dark:text-gray-300 mt-6">+ The following error occurred when fetching your Google Analytics data.+ </p><p class="text-red-700 font-medium mt-3"><%= error %></p>- <% end %>+ <% end %>- <%= submit "Confirm import", class: "button mt-6" %>+ <%= submit("Confirm import", class: "button mt-6") %><% end %>diff --git a/lib/plausible_web/templates/universal_analytics/user_metric_form.html.heex b/lib/plausible_web/templates/universal_analytics/user_metric_form.html.heexnew file mode 100644index 0000000000..2f353f984b--- /dev/null+++ b/lib/plausible_web/templates/universal_analytics/user_metric_form.html.heex@@ -0,0 +1,46 @@+<div class="max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8">+ <h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2>++ <div class="mt-6 text-sm text-gray-500 dark:text-gray-200">+ <p>+ <svg+ xmlns="http://www.w3.org/2000/svg"+ class="h-4 w-4 inline text-orange-600"+ fill="none"+ viewBox="0 0 24 24"+ stroke="currentColor"+ stroke-width="2"+ >+ <path+ stroke-linecap="round"+ stroke-linejoin="round"+ d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-3L13.732 4c-.77-1.333-2.694-1.333-3.464 0L3.34 16c-.77 1.333.192 3 1.732 3z"+ />+ </svg>+ Important: Since your GA property includes data from before 23rd August 2016, you have to take an extra step to make sure we can import data smoothly.+ </p>++ <ol class="mt-4">+ <li>1. Navigate to the GA property you want to import from</li>+ <li>2. Go to Admin > Property Settings > User Analysis</li>+ <li>3. Make sure <i>Enable Users Metric in Reporting</i> is <b>OFF</b></li>+ </ol>++ <p class="mt-4">+ The setting may take a few minutes to take effect. If your imported data is showing 0 visitors in unexpected places, it's probably caused by this and you+ can try importing again later.+ </p>+ </div>++ <%= link("Continue ->",+ to:+ Routes.universal_analytics_path(@conn, :confirm, @site.domain,+ view_id: @view_id,+ access_token: @access_token,+ refresh_token: @refresh_token,+ expires_at: @expires_at,+ legacy: @legacy+ ),+ class: "button mt-6"+ ) %>+</div>diff --git a/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex b/lib/plausible_web/templates/universal_analytics/view_id_form.html.heexsimilarity index 66%rename from lib/plausible_web/templates/site/import_from_google_view_id_form.html.eexrename to lib/plausible_web/templates/universal_analytics/view_id_form.html.heexindex d8b8db4442..e9e371f75c 100644--- a/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex+++ b/lib/plausible_web/templates/universal_analytics/view_id_form.html.heex@@ -1,4 +1,4 @@-<%= form_for @conn, Routes.site_path(@conn, :import_from_google_view_id, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %>+<%= form_for @conn, Routes.universal_analytics_path(@conn, :view_id, @site.domain), [class: "max-w-md w-full mx-auto bg-white dark:bg-gray-800 shadow-md rounded px-8 pt-6 pb-8 mb-4 mt-8"], fn f -> %><h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2><%= hidden_input(f, :access_token, value: @access_token) %>@@ -12,9 +12,9 @@<div class="mt-3"><%= styled_label(f, :view_id, "Google Analytics view") %>- <%= styled_select f, :view_id, @view_ids, prompt: "(Choose view)", required: "true" %>+ <%= styled_select(f, :view_id, @view_ids, prompt: "(Choose view)", required: "true") %><%= styled_error(@conn.assigns[:selected_view_id_error]) %></div>- <%= submit "Continue ->", class: "button mt-6" %>+ <%= submit("Continue ->", class: "button mt-6") %><% end %>diff --git a/lib/plausible_web/views/google_analytics4_view.ex b/lib/plausible_web/views/google_analytics4_view.exnew file mode 100644index 0000000000..760309b804--- /dev/null+++ b/lib/plausible_web/views/google_analytics4_view.ex@@ -0,0 +1,4 @@+defmodule PlausibleWeb.GoogleAnalytics4View do+ use PlausibleWeb, :view+ use Plausible+enddiff --git a/lib/plausible_web/views/universal_analytics_view.ex b/lib/plausible_web/views/universal_analytics_view.exnew file mode 100644index 0000000000..f95899ee77--- /dev/null+++ b/lib/plausible_web/views/universal_analytics_view.ex@@ -0,0 +1,4 @@+defmodule PlausibleWeb.UniversalAnalyticsView do+ use PlausibleWeb, :view+ use Plausible+enddiff --git a/test/plausible/google/api_test.exs b/test/plausible/google/api_test.exsindex c448e7c431..076a1af469 100644--- a/test/plausible/google/api_test.exs+++ b/test/plausible/google/api_test.exs@@ -1,8 +1,8 @@-defmodule Plausible.Google.ApiTest do+defmodule Plausible.Google.APITest douse Plausible.DataCase, async: trueuse Plausible.Test.Support.HTTPMocker- alias Plausible.Google.Api+ alias Plausible.Googlealias Plausible.Imported.UniversalAnalyticsimport ExUnit.CaptureLog@@ -44,7 +44,7 @@ defmodule Plausible.Google.ApiTest doPlausible.Imported.Buffer.insert_many(buffer, table, records)end- assert :ok == Plausible.Google.Api.import_analytics(date_range, view_id, auth, persist_fn)+ assert :ok == Google.UA.API.import_analytics(date_range, view_id, auth, persist_fn)Plausible.Imported.Buffer.flush(buffer)Plausible.Imported.Buffer.stop(buffer)@@ -86,7 +86,7 @@ defmodule Plausible.Google.ApiTest doPlausible.Imported.Buffer.insert_many(buffer, table, records)end- assert :ok == Plausible.Google.Api.import_analytics(range, "123551", auth, persist_fn)+ assert :ok == Google.UA.API.import_analytics(range, "123551", auth, persist_fn)Plausible.Imported.Buffer.flush(buffer)Plausible.Imported.Buffer.stop(buffer)@@ -98,7 +98,7 @@ defmodule Plausible.Google.ApiTest do@tag :slowtest "will fetch and persist import data from Google Analytics" do- request = %Plausible.Google.ReportRequest{+ request = %Plausible.Google.UA.ReportRequest{dataset: "imported_exit_pages",view_id: "123",date_range: Date.range(~D[2022-01-01], ~D[2022-02-01]),@@ -127,7 +127,7 @@ defmodule Plausible.Google.ApiTest dohideValueRanges: true,metrics: [%{expression: "ga:users"}, %{expression: "ga:exits"}],orderBys: [%{fieldName: "ga:date", sortOrder: "DESCENDING"}],- pageSize: 10000,+ pageSize: 10_000,pageToken: nil,viewId: "123"}@@ -139,7 +139,7 @@ defmodule Plausible.Google.ApiTest do)assert :ok =- Api.fetch_and_persist(request,+ Google.UA.API.fetch_and_persist(request,sleep_time: 0,persist_fn: fn dataset, row ->assert dataset == "imported_exit_pages"@@ -167,7 +167,7 @@ defmodule Plausible.Google.ApiTest doend)- request = %Plausible.Google.ReportRequest{+ request = %Plausible.Google.UA.ReportRequest{view_id: "123",date_range: Date.range(~D[2022-01-01], ~D[2022-02-01]),dimensions: ["ga:date"],@@ -178,7 +178,7 @@ defmodule Plausible.Google.ApiTest do}assert {:error, :request_failed} =- Api.fetch_and_persist(request,+ Google.UA.API.fetch_and_persist(request,sleep_time: 0,persist_fn: fn _dataset, _rows -> :ok end)@@ -197,7 +197,7 @@ defmodule Plausible.Google.ApiTest doend)- request = %Plausible.Google.ReportRequest{+ request = %Plausible.Google.UA.ReportRequest{dataset: "imported_exit_pages",view_id: "123",date_range: Date.range(~D[2022-01-01], ~D[2022-02-01]),@@ -209,7 +209,7 @@ defmodule Plausible.Google.ApiTest do}assert :ok ==- Api.fetch_and_persist(request,+ Google.UA.API.fetch_and_persist(request,sleep_time: 0,persist_fn: fn dataset, rows ->assert dataset == "imported_exit_pages"@@ -253,7 +253,7 @@ defmodule Plausible.Google.ApiTest doquery = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}- assert {:error, "google_auth_error"} = Plausible.Google.Api.fetch_stats(site, query, 5)+ assert {:error, "google_auth_error"} = Google.API.fetch_stats(site, query, 5)endtest "returns whatever error code google returns on API client error", %{site: site} do@@ -270,7 +270,7 @@ defmodule Plausible.Google.ApiTest doquery = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}- assert {:error, "some_error"} = Plausible.Google.Api.fetch_stats(site, query, 5)+ assert {:error, "some_error"} = Google.API.fetch_stats(site, query, 5)endtest "returns generic HTTP error and logs it", %{site: site} do@@ -290,7 +290,7 @@ defmodule Plausible.Google.ApiTest dolog =capture_log(fn ->assert {:error, "failed_to_list_stats"} =- Plausible.Google.Api.fetch_stats(site, query, 5)+ Google.API.fetch_stats(site, query, 5)end)assert log =~ "Google Analytics: failed to list stats: %Finch.Error{reason: :some_reason}"@@ -314,7 +314,7 @@ defmodule Plausible.Google.ApiTest do[%{name: ["keyword1", "keyword2"], visitors: 25},%{name: ["keyword3", "keyword4"], visitors: 15}- ]} = Plausible.Google.Api.fetch_stats(site, query, 5)+ ]} = Google.API.fetch_stats(site, query, 5)endtest "returns next page when page argument is set", %{user: user, site: site} do@@ -336,7 +336,7 @@ defmodule Plausible.Google.ApiTest do[%{name: ["keyword1", "keyword2"], visitors: 25},%{name: ["keyword3", "keyword4"], visitors: 15}- ]} = Plausible.Google.Api.fetch_stats(site, query, 5)+ ]} = Google.API.fetch_stats(site, query, 5)endtest "defaults first page when page argument is not set", %{user: user, site: site} do@@ -355,7 +355,7 @@ defmodule Plausible.Google.ApiTest do[%{name: ["keyword1", "keyword2"], visitors: 25},%{name: ["keyword3", "keyword4"], visitors: 15}- ]} = Plausible.Google.Api.fetch_stats(site, query, 5)+ ]} = Google.API.fetch_stats(site, query, 5)endtest "returns error when token refresh fails", %{user: user, site: site} do@@ -372,7 +372,7 @@ defmodule Plausible.Google.ApiTest doquery = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}- assert {:error, "invalid_grant"} = Plausible.Google.Api.fetch_stats(site, query, 5)+ assert {:error, "invalid_grant"} = Google.API.fetch_stats(site, query, 5)endend@@ -393,7 +393,7 @@ defmodule Plausible.Google.ApiTest do%{"one.test" => [{"57238190 - one.test", "57238190"}],"two.test" => [{"54460083 - two.test", "54460083"}]- }} == Plausible.Google.Api.list_views("access_token")+ }} == Google.UA.API.list_views("access_token")endtest "list_views/1 returns authentication_failed when request fails with HTTP 403" do@@ -405,7 +405,7 @@ defmodule Plausible.Google.ApiTest doend)- assert {:error, :authentication_failed} == Plausible.Google.Api.list_views("access_token")+ assert {:error, :authentication_failed} == Google.UA.API.list_views("access_token")endtest "list_views/1 returns authentication_failed when request fails with HTTP 401" do@@ -417,7 +417,7 @@ defmodule Plausible.Google.ApiTest doend)- assert {:error, :authentication_failed} == Plausible.Google.Api.list_views("access_token")+ assert {:error, :authentication_failed} == Google.UA.API.list_views("access_token")endtest "list_views/1 returns error when request fails with HTTP 500" do@@ -429,6 +429,6 @@ defmodule Plausible.Google.ApiTest doend)- assert {:error, :unknown} == Plausible.Google.Api.list_views("access_token")+ assert {:error, :unknown} == Google.UA.API.list_views("access_token")endenddiff --git a/test/plausible/google/ga4/api_test.exs b/test/plausible/google/ga4/api_test.exsnew file mode 100644index 0000000000..029c46ff7c--- /dev/null+++ b/test/plausible/google/ga4/api_test.exs@@ -0,0 +1,57 @@+defmodule Plausible.Google.GA4.APITest do+ use Plausible.DataCase, async: true++ import Mox++ alias Plausible.Google.GA4++ setup :verify_on_exit!++ describe "list_properties/1" do+ test "returns list of properties grouped by accounts" do+ result = Jason.decode!(File.read!("fixture/ga4_list_properties.json"))++ expect(Plausible.HTTPClient.Mock, :get, fn _url, _opts ->+ {:ok, %Finch.Response{status: 200, body: result}}+ end)++ assert {:ok, accounts} = GA4.API.list_properties("some_access_token")++ assert [+ {"account.one (accounts/28425178)",+ [{"account.one - GA4 (properties/428685906)", "properties/428685906"}]},+ {"Demo Account (accounts/54516992)",+ [+ {"GA4 - Flood-It! (properties/153293282)", "properties/153293282"},+ {"GA4 - Google Merch Shop (properties/213025502)", "properties/213025502"}+ ]}+ ] = accounts+ end+ end++ describe "get_property/2" do+ test "returns tuple consisting of display name and value of a property" do+ result = Jason.decode!(File.read!("fixture/ga4_list_properties.json"))++ expect(Plausible.HTTPClient.Mock, :get, fn _url, _opts ->+ {:ok, %Finch.Response{status: 200, body: result}}+ end)++ assert {:ok, {"GA4 - Flood-It! (properties/153293282)", "properties/153293282"}} =+ GA4.API.get_property("some_access_token", "properties/153293282")+ end+ end++ describe "get_analytics_start_date/2" do+ test "returns stats start date for a given property" do+ result = Jason.decode!(File.read!("fixture/ga4_start_date.json"))++ expect(Plausible.HTTPClient.Mock, :post, fn _url, _headers, _body ->+ {:ok, %Finch.Response{status: 200, body: result}}+ end)++ assert {:ok, ~D[2024-02-22]} =+ GA4.API.get_analytics_start_date("some_access_token", "properties/153293282")+ end+ end+enddiff --git a/test/plausible/imported/google_analytics4_test.exs b/test/plausible/imported/google_analytics4_test.exsnew file mode 100644index 0000000000..20e0384405--- /dev/null+++ b/test/plausible/imported/google_analytics4_test.exs@@ -0,0 +1,98 @@+defmodule Plausible.Imported.GoogleAnalytics4Test do+ use Plausible.DataCase, async: true++ import Mox++ import Ecto.Query, only: [from: 2]++ alias Plausible.Imported.GoogleAnalytics4++ @refresh_token_body Jason.decode!(File.read!("fixture/ga_refresh_token.json"))++ @full_report_mock [+ "fixture/ga4_report_imported_visitors.json",+ "fixture/ga4_report_imported_sources.json",+ "fixture/ga4_report_imported_pages.json",+ "fixture/ga4_report_imported_entry_pages.json",+ "fixture/ga4_report_imported_locations.json",+ "fixture/ga4_report_imported_devices.json",+ "fixture/ga4_report_imported_browsers.json",+ "fixture/ga4_report_imported_operating_systems.json"+ ]+ |> Enum.map(&File.read!/1)+ |> Enum.map(&Jason.decode!/1)++ setup :verify_on_exit!++ describe "parse_args/1 and import_data/2" do+ setup [:create_user, :create_new_site]++ test "imports data returned from GA4 Data API", %{user: user, site: site} do+ past = DateTime.add(DateTime.utc_now(), -3600, :second)++ {:ok, job} =+ Plausible.Imported.GoogleAnalytics4.new_import(+ site,+ user,+ property: "properties/123456",+ start_date: ~D[2024-02-20],+ end_date: Date.utc_today(),+ access_token: "redacted_access_token",+ refresh_token: "redacted_refresh_token",+ token_expires_at: DateTime.to_iso8601(past)+ )++ site_import = Plausible.Imported.get_import(job.args.import_id)++ opts = job |> Repo.reload!() |> Map.get(:args) |> GoogleAnalytics4.parse_args()++ opts = Keyword.put(opts, :flush_interval_ms, 10)++ expect(Plausible.HTTPClient.Mock, :post, fn "https://www.googleapis.com/oauth2/v4/token",+ headers,+ body ->+ assert [{"content-type", "application/x-www-form-urlencoded"}] == headers++ assert %{+ grant_type: :refresh_token,+ redirect_uri: "http://localhost:8000/auth/google/callback",+ refresh_token: "redacted_refresh_token"+ } = body++ {:ok, %Finch.Response{status: 200, body: @refresh_token_body}}+ end)++ for report <- @full_report_mock do+ expect(Plausible.HTTPClient.Mock, :post, fn _url, headers, _body, _opts ->+ assert [{"Authorization", "Bearer 1/fFAGRNJru1FTz70BzhT3Zg"}] == headers+ {:ok, %Finch.Response{status: 200, body: report}}+ end)+ end++ Enum.each(Plausible.Imported.tables(), fn table ->+ query = from(imported in table, where: imported.site_id == ^site.id)+ assert await_clickhouse_count(query, 0)+ end)++ assert :ok = GoogleAnalytics4.import_data(site_import, opts)++ Enum.each(Plausible.Imported.tables(), fn table ->+ count =+ case table do+ "imported_sources" -> 3+ "imported_visitors" -> 3+ "imported_pages" -> 8+ "imported_entry_pages" -> 4+ "imported_exit_pages" -> 0+ "imported_locations" -> 4+ "imported_devices" -> 4+ "imported_browsers" -> 5+ "imported_operating_systems" -> 4+ end++ query = from(imported in table, where: imported.site_id == ^site.id)+ assert await_clickhouse_count(query, count)+ end)+ end+ end+enddiff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exsindex d26395638e..45f1c02381 100644--- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs@@ -1461,7 +1461,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do])conn = get(conn, "/api/stats/#{site.domain}/referrers/Google?period=day")- {:ok, terms} = Plausible.Google.Api.Mock.fetch_stats(nil, nil, nil)+ {:ok, terms} = Plausible.Google.API.Mock.fetch_stats(nil, nil, nil)assert json_response(conn, 200) == %{"total_visitors" => 2,diff --git a/test/support/google_api_mock.ex b/test/support/google_api_mock.exindex 5aa33a7180..878cbec970 100644--- a/test/support/google_api_mock.ex+++ b/test/support/google_api_mock.ex@@ -1,4 +1,8 @@-defmodule Plausible.Google.Api.Mock do+defmodule Plausible.Google.API.Mock do+ @moduledoc """+ Mock of API to Google services.+ """+def fetch_stats(_auth, _query, _limit) do{:ok,[