Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Refresh Google Analytics token before import (#2254)
Vinicius Brasil1adda42a75c9
diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.exindex 85531658d9..7878bc53f6 100644--- a/lib/plausible/google/api.ex+++ b/lib/plausible/google/api.ex@@ -30,8 +30,8 @@ defmodule Plausible.Google.Api doenddef fetch_verified_properties(auth) do- with {:ok, auth} <- refresh_if_needed(auth),- {:ok, sites} <- Plausible.Google.HTTP.list_sites(auth.access_token) do+ with {:ok, access_token} <- maybe_refresh_token(auth),+ {:ok, sites} <- Plausible.Google.HTTP.list_sites(access_token) dosites|> Map.get("siteEntry", [])|> Enum.filter(fn site -> site["permissionLevel"] in @verified_permission_levels end)@@ -43,10 +43,15 @@ defmodule Plausible.Google.Api dodef fetch_stats(site, %{filters: %{} = filters, date_range: date_range}, limit) dowith site <- Plausible.Repo.preload(site, :google_auth),- {:ok, %{access_token: access_token, property: property}} <-- refresh_if_needed(site.google_auth),+ {:ok, access_token} <- maybe_refresh_token(site.google_auth),{:ok, stats} <-- HTTP.list_stats(access_token, property, date_range, limit, filters["page"]) do+ HTTP.list_stats(+ access_token,+ site.google_auth.property,+ date_range,+ limit,+ filters["page"]+ ) dostats|> Map.get("rows", [])|> Enum.filter(fn row -> row["clicks"] > 0 end)@@ -103,9 +108,15 @@ defmodule Plausible.Google.Api doendend+ @type import_auth :: {+ access_token :: String.t(),+ refresh_token :: String.t(),+ expires_at :: String.t()+ }+@per_page 10_000@max_attempts 5- @spec import_analytics(Plausible.Site.t(), Date.Range.t(), String.t(), String.t()) ::+ @spec import_analytics(Plausible.Site.t(), Date.Range.t(), String.t(), import_auth()) :::ok | {:error, term()}@doc """Imports stats from a Google Analytics UA view to a Plausible site.@@ -125,7 +136,21 @@ defmodule Plausible.Google.Api do- [GA Dimensions reference](https://ga-dev-tools.web.app/dimensions-metrics-explorer)"""- def import_analytics(site, date_range, view_id, access_token) do+ def import_analytics(site, date_range, view_id, auth) do+ with {:ok, access_token} <- maybe_refresh_token(auth),+ :ok <- do_import_analytics(site, date_range, view_id, access_token) do+ :ok+ else+ {:error, cause} ->+ Sentry.capture_message("Failed to import from Google Analytics",+ extra: %{site: site.domain, error: inspect(cause)}+ )++ {:error, cause}+ end+ end++ defp do_import_analytics(site, date_range, view_id, access_token) do{:ok, buffer} = Plausible.Google.Buffer.start_link()result =@@ -176,10 +201,6 @@ defmodule Plausible.Google.Api do{:error, cause} ->if attempt >= @max_attempts do- Sentry.capture_message("Failed to import from Google Analytics",- extra: %{site: site.domain, error: inspect(cause)}- )-{:error, cause}elseProcess.sleep(sleep_time)@@ -188,28 +209,56 @@ defmodule Plausible.Google.Api doendend- defp refresh_if_needed(auth) do- if Timex.before?(auth.expires, Timex.now() |> Timex.shift(seconds: 30)) do- do_refresh_token(auth)+ defp maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do+ with true <- needs_to_refresh_token?(auth.expires),+ {:ok, {new_access_token, expires_at}} <- do_refresh_token(auth.refresh_token),+ changeset <-+ Plausible.Site.GoogleAuth.changeset(auth, %{+ access_token: new_access_token,+ expires: expires_at+ }),+ {:ok, _google_auth} <- Plausible.Repo.update(changeset) do+ {:ok, new_access_token}else- {:ok, auth}+ false -> {:ok, auth.access_token}+ {:error, cause} -> {:error, cause}endend- defp do_refresh_token(auth) do- case HTTP.refresh_auth_token(auth.refresh_token) do- {:ok, %{"access_token" => access_token, "expires_in" => expires_in}} ->- expires_in = NaiveDateTime.add(NaiveDateTime.utc_now(), expires_in)+ defp maybe_refresh_token({access_token, nil, nil}) do+ {:ok, access_token}+ end++ defp maybe_refresh_token({access_token, refresh_token, expires_at}) do+ if needs_to_refresh_token?(expires_at) do+ do_refresh_token(refresh_token)+ else+ {:ok, access_token}+ end+ end- auth- |> Plausible.Site.GoogleAuth.changeset(%{access_token: access_token, expires: expires_in})- |> Plausible.Repo.update()+ defp do_refresh_token(refresh_token) do+ case HTTP.refresh_auth_token(refresh_token) do+ {:ok, %{"access_token" => new_access_token, "expires_at" => expires_in}} ->+ expires_at = NaiveDateTime.add(NaiveDateTime.utc_now(), expires_in)+ {:ok, {new_access_token, expires_at}}- error ->- error+ {:error, cause} ->+ {:error, cause}endend+ defp needs_to_refresh_token?(expires_at) when is_binary(expires_at) do+ expires_at+ |> NaiveDateTime.from_iso8601!()+ |> needs_to_refresh_token?()+ end++ defp needs_to_refresh_token?(%NaiveDateTime{} = expires_at) do+ thirty_seconds_ago = Timex.shift(Timex.now(), seconds: 30)+ Timex.before?(expires_at, thirty_seconds_ago)+ end+defp client_id() doKeyword.fetch!(Application.get_env(:plausible, :google), :client_id)enddiff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.exindex 29d9172a64..0dd6bfa429 100644--- a/lib/plausible_web/controllers/auth_controller.ex+++ b/lib/plausible_web/controllers/auth_controller.ex@@ -546,13 +546,16 @@ defmodule PlausibleWeb.AuthController dores = Plausible.Google.HTTP.fetch_access_token(code)[site_id, redirect_to] = Jason.decode!(state)site = Repo.get(Plausible.Site, site_id)+ expires_at = NaiveDateTime.add(NaiveDateTime.utc_now(), res["expires_in"])case redirect_to do"import" ->redirect(conn,to:Routes.site_path(conn, :import_from_google_view_id_form, site.domain,- access_token: res["access_token"]+ access_token: res["access_token"],+ refresh_token: res["refresh_token"],+ expires_at: NaiveDateTime.to_iso8601(expires_at)))@@ -565,7 +568,7 @@ defmodule PlausibleWeb.AuthController doemail: id["email"],refresh_token: res["refresh_token"],access_token: res["access_token"],- expires: NaiveDateTime.utc_now() |> NaiveDateTime.add(res["expires_in"]),+ expires_at: expires_at,user_id: conn.assigns[:current_user].id,site_id: site_id})diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.exindex 1dc855af53..d22af55b97 100644--- a/lib/plausible_web/controllers/site_controller.ex+++ b/lib/plausible_web/controllers/site_controller.ex@@ -636,7 +636,9 @@ defmodule PlausibleWeb.SiteController dodef import_from_google_user_metric_notice(conn, %{"view_id" => view_id,- "access_token" => access_token+ "access_token" => access_token,+ "refresh_token" => refresh_token,+ "expires_at" => expires_at}) dosite = conn.assigns[:site]@@ -646,11 +648,17 @@ defmodule PlausibleWeb.SiteController dosite: site,view_id: view_id,access_token: access_token,+ refresh_token: refresh_token,+ expires_at: expires_at,layout: {PlausibleWeb.LayoutView, "focus.html"})end- def import_from_google_view_id_form(conn, %{"access_token" => access_token}) do+ def import_from_google_view_id_form(conn, %{+ "access_token" => access_token,+ "refresh_token" => refresh_token,+ "expires_at" => expires_at+ }) dosite = conn.assigns[:site]view_ids = Plausible.Google.Api.list_views(access_token)@@ -658,6 +666,8 @@ defmodule PlausibleWeb.SiteController do|> assign(:skip_plausible_tracking, true)|> render("import_from_google_view_id_form.html",access_token: access_token,+ refresh_token: refresh_token,+ expires_at: expires_at,site: site,view_ids: view_ids,layout: {PlausibleWeb.LayoutView, "focus.html"}@@ -666,7 +676,12 @@ defmodule PlausibleWeb.SiteController do# see https://stackoverflow.com/a/57416769@google_analytics_new_user_metric_date ~D[2016-08-24]- def import_from_google_view_id(conn, %{"view_id" => view_id, "access_token" => access_token}) do+ def import_from_google_view_id(conn, %{+ "view_id" => view_id,+ "access_token" => access_token,+ "refresh_token" => refresh_token,+ "expires_at" => expires_at+ }) dosite = conn.assigns[:site]start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token)@@ -679,6 +694,8 @@ defmodule PlausibleWeb.SiteController do|> assign(:skip_plausible_tracking, true)|> render("import_from_google_view_id_form.html",access_token: access_token,+ refresh_token: refresh_token,+ expires_at: expires_at,site: site,view_ids: view_ids,selected_view_id_error: "No data found. Nothing to import",@@ -691,7 +708,9 @@ defmodule PlausibleWeb.SiteController doto:Routes.site_path(conn, :import_from_google_user_metric_notice, site.domain,view_id: view_id,- access_token: access_token+ access_token: access_token,+ refresh_token: refresh_token,+ expires_at: expires_at))else@@ -699,14 +718,21 @@ defmodule PlausibleWeb.SiteController doto:Routes.site_path(conn, :import_from_google_confirm, site.domain,view_id: view_id,- access_token: access_token+ access_token: access_token,+ refresh_token: refresh_token,+ expires_at: expires_at))endendend- def import_from_google_confirm(conn, %{"access_token" => access_token, "view_id" => view_id}) do+ def import_from_google_confirm(conn, %{+ "view_id" => view_id,+ "access_token" => access_token,+ "refresh_token" => refresh_token,+ "expires_at" => expires_at+ }) dosite = conn.assigns[:site]start_date = Plausible.Google.HTTP.get_analytics_start_date(view_id, access_token)@@ -720,6 +746,8 @@ defmodule PlausibleWeb.SiteController do|> assign(:skip_plausible_tracking, true)|> render("import_from_google_confirm.html",access_token: access_token,+ refresh_token: refresh_token,+ expires_at: expires_at,site: site,selected_view_id: view_id,selected_view_id_name: view_name,@@ -733,7 +761,9 @@ defmodule PlausibleWeb.SiteController do"view_id" => view_id,"start_date" => start_date,"end_date" => end_date,- "access_token" => access_token+ "access_token" => access_token,+ "refresh_token" => refresh_token,+ "expires_at" => expires_at}) dosite = conn.assigns[:site]@@ -743,7 +773,9 @@ defmodule PlausibleWeb.SiteController do"view_id" => view_id,"start_date" => start_date,"end_date" => end_date,- "access_token" => access_token+ "access_token" => access_token,+ "refresh_token" => refresh_token,+ "token_expires_at" => expires_at})Ecto.Multi.new()diff --git a/lib/plausible_web/templates/site/import_from_google_confirm.html.eex b/lib/plausible_web/templates/site/import_from_google_confirm.html.eexindex 072d998d1e..4e1018faa3 100644--- a/lib/plausible_web/templates/site/import_from_google_confirm.html.eex+++ b/lib/plausible_web/templates/site/import_from_google_confirm.html.eex@@ -2,6 +2,8 @@<h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2><%= hidden_input(f, :access_token, value: @access_token) %>+ <%= hidden_input(f, :refresh_token, value: @refresh_token) %>+ <%= hidden_input(f, :expires_at, value: @expires_at) %><%= case @start_date do %><% {:ok, start_date} -> %>diff --git a/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex b/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eexindex c303701f4d..d8f69f2e5e 100644--- a/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex+++ b/lib/plausible_web/templates/site/import_from_google_user_metric_form.html.eex@@ -22,5 +22,5 @@</p></div>- <%= link("Continue ->", to: Routes.site_path(@conn, :import_from_google_confirm, @site.domain, view_id: @view_id, access_token: @access_token), class: "button mt-6") %>+ <%= link("Continue ->", to: Routes.site_path(@conn, :import_from_google_confirm, @site.domain, view_id: @view_id, access_token: @access_token, refresh_token: @refresh_token, expires_at: @expires_at), class: "button mt-6") %></div>diff --git a/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex b/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eexindex e6e9df24ac..bd8bbee540 100644--- a/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex+++ b/lib/plausible_web/templates/site/import_from_google_view_id_form.html.eex@@ -2,6 +2,8 @@<h2 class="text-xl font-black dark:text-gray-100">Import from Google Analytics</h2><%= hidden_input(f, :access_token, value: @access_token) %>+ <%= hidden_input(f, :refresh_token, value: @refresh_token) %>+ <%= hidden_input(f, :expires_at, value: @expires_at) %><%= case @view_ids do %><% {:ok, view_ids} -> %>diff --git a/lib/workers/import_google_analytics.ex b/lib/workers/import_google_analytics.exindex df1dbeae13..0fc9043748 100644--- a/lib/workers/import_google_analytics.ex+++ b/lib/workers/import_google_analytics.ex@@ -9,13 +9,13 @@ defmodule Plausible.Workers.ImportGoogleAnalytics do@impl Oban.Workerdef perform(%Oban.Job{- args: %{- "site_id" => site_id,- "view_id" => view_id,- "start_date" => start_date,- "end_date" => end_date,- "access_token" => access_token- }+ args:+ %{+ "site_id" => site_id,+ "view_id" => view_id,+ "start_date" => start_date,+ "end_date" => end_date+ } = args},google_api \\ Plausible.Google.Api) do@@ -24,7 +24,9 @@ defmodule Plausible.Workers.ImportGoogleAnalytics doend_date = Date.from_iso8601!(end_date)date_range = Date.range(start_date, end_date)- case google_api.import_analytics(site, date_range, view_id, access_token) do+ auth = {args["access_token"], args["refresh_token"], args["token_expires_at"]}++ case google_api.import_analytics(site, date_range, view_id, auth) do:ok ->Plausible.Site.import_success(site)|> Repo.update!()diff --git a/test/plausible/google/vcr_test.exs b/test/plausible/google/vcr_test.exsindex cb41b6ddf0..f93fe385d6 100644--- a/test/plausible/google/vcr_test.exs+++ b/test/plausible/google/vcr_test.exs@@ -21,11 +21,13 @@ defmodule Plausible.Google.Api.VCRTest doinserts_before_importing = get_insert_count()before_importing_timestamp = DateTime.utc_now()- access_token = "***"view_id = "54297898"date_range = Date.range(~D[2011-01-01], ~D[2022-07-19])- assert :ok == Plausible.Google.Api.import_analytics(site, date_range, view_id, access_token)+ future = DateTime.utc_now() |> DateTime.add(3600, :second) |> DateTime.to_iso8601()+ auth = {"***", "refresh_token", future}++ assert :ok == Plausible.Google.Api.import_analytics(site, date_range, view_id, auth)total_seconds = DateTime.diff(DateTime.utc_now(), before_importing_timestamp, :second)total_inserts = get_insert_count() - inserts_before_importingdiff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exsindex b4dc35ed8b..f0d0090643 100644--- a/test/plausible_web/controllers/site_controller_test.exs+++ b/test/plausible_web/controllers/site_controller_test.exs@@ -745,7 +745,11 @@ defmodule PlausibleWeb.SiteControllerTest doresponse =conn- |> get("/#{site.domain}/import/google-analytics/view-id", %{"access_token" => "token"})+ |> get("/#{site.domain}/import/google-analytics/view-id", %{+ "access_token" => "token",+ "refresh_token" => "foo",+ "expires_at" => "2022-09-22T20:01:37.112777"+ })|> html_response(200)assert response =~ "57238190 - one.test"@@ -761,7 +765,9 @@ defmodule PlausibleWeb.SiteControllerTest do"view_id" => "123","start_date" => "2018-03-01","end_date" => "2022-03-01",- "access_token" => "token"+ "access_token" => "token",+ "refresh_token" => "foo",+ "expires_at" => "2022-09-22T20:01:37.112777"})imported_data = Repo.reload(site).imported_data@@ -777,7 +783,9 @@ defmodule PlausibleWeb.SiteControllerTest do"view_id" => "123","start_date" => "2018-03-01","end_date" => "2022-03-01",- "access_token" => "token"+ "access_token" => "token",+ "refresh_token" => "foo",+ "expires_at" => "2022-09-22T20:01:37.112777"})assert_enqueued(@@ -787,7 +795,9 @@ defmodule PlausibleWeb.SiteControllerTest do"view_id" => "123","start_date" => "2018-03-01","end_date" => "2022-03-01",- "access_token" => "token"+ "access_token" => "token",+ "refresh_token" => "foo",+ "token_expires_at" => "2022-09-22T20:01:37.112777"})end