Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Improve GA4 import error handling (#4714)
Adrian Gruntkowskibe83b7829df7
diff --git a/lib/plausible/google/ga4/api.ex b/lib/plausible/google/ga4/api.exindex 64da14ba0f..7299149814 100644--- a/lib/plausible/google/ga4/api.ex+++ b/lib/plausible/google/ga4/api.ex@@ -73,6 +73,7 @@ defmodule Plausible.Google.GA4.API dodef import_analytics(date_range, property, auth, opts) dopersist_fn = Keyword.fetch!(opts, :persist_fn)+ fetch_opts = Keyword.get(opts, :fetch_opts, [])resume_opts = Keyword.get(opts, :resume_opts, [])Logger.debug(@@ -80,11 +81,18 @@ defmodule Plausible.Google.GA4.API do)with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do- do_import_analytics(date_range, property, access_token, persist_fn, resume_opts)+ do_import_analytics(date_range, property, access_token, persist_fn, fetch_opts, resume_opts)endend- defp do_import_analytics(date_range, property, access_token, persist_fn, [] = _resume_opts) do+ defp do_import_analytics(+ date_range,+ property,+ access_token,+ persist_fn,+ fetch_opts,+ [] = _resume_opts+ ) doEnum.reduce_while(GA4.ReportRequest.full_report(), :ok, fn report_request, :ok ->Logger.debug("[#{inspect(__MODULE__)}:#{property}] Starting to import #{report_request.dataset}"@@ -92,14 +100,21 @@ defmodule Plausible.Google.GA4.API doreport_request = prepare_request(report_request, date_range, property, access_token)- case fetch_and_persist(report_request, persist_fn: persist_fn) do+ case fetch_and_persist(report_request, persist_fn: persist_fn, fetch_opts: fetch_opts) do:ok -> {:cont, :ok}{:error, _} = error -> {:halt, error}endend)end- defp do_import_analytics(date_range, property, access_token, persist_fn, resume_opts) do+ defp do_import_analytics(+ date_range,+ property,+ access_token,+ persist_fn,+ fetch_opts,+ resume_opts+ ) dodataset = Keyword.fetch!(resume_opts, :dataset)offset = Keyword.fetch!(resume_opts, :offset)@@ -122,7 +137,7 @@ defmodule Plausible.Google.GA4.API do|> prepare_request(date_range, property, access_token)|> Map.put(:offset, request_offset)- case fetch_and_persist(report_request, persist_fn: persist_fn) do+ case fetch_and_persist(report_request, persist_fn: persist_fn, fetch_opts: fetch_opts) do:ok -> {:cont, :ok}{:error, _} = error -> {:halt, error}end@@ -134,7 +149,9 @@ defmodule Plausible.Google.GA4.API dodef fetch_and_persist(%GA4.ReportRequest{} = report_request, opts \\ []) dopersist_fn = Keyword.fetch!(opts, :persist_fn)attempt = Keyword.get(opts, :attempt, 1)- sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)+ fetch_opts = Keyword.get(opts, :fetch_opts, [])+ max_attempts = Keyword.get(fetch_opts, :max_attempts, @max_attempts)+ sleep_time = Keyword.get(fetch_opts, :sleep_time, @backoff_factor)case GA4.HTTP.get_report(report_request) do{:ok, {_, 0}} ->@@ -168,7 +185,7 @@ defmodule Plausible.Google.GA4.API do{:error, {:rate_limit_exceeded, details}}{:error, cause} ->- if attempt >= @max_attempts do+ if attempt >= max_attempts doLogger.debug("[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Terminating.")diff --git a/lib/plausible/google/ga4/http.ex b/lib/plausible/google/ga4/http.exindex 68533a917a..437fd1ed10 100644--- a/lib/plausible/google/ga4/http.ex+++ b/lib/plausible/google/ga4/http.ex@@ -69,24 +69,38 @@ defmodule Plausible.Google.GA4.HTTP do{:error,{:rate_limit_exceeded, dataset: report_request.dataset, offset: report_request.offset}}+ {:error, %{reason: %{status: status, body: body}} = error} when status >= 500 ->+ log_ce_error("retrieving report for #{report_request.dataset}", error)++ Logger.warning(+ "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} with code #{status}: #{inspect(body)}"+ )++ Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})++ {:error, {:server_failed, dataset: report_request.dataset, offset: report_request.offset}}+{:error, %{reason: %{status: status, body: body}} = error} ->log_ce_error("retrieving report for #{report_request.dataset}", error)- Logger.debug(+ Logger.warning("[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} with code #{status}: #{inspect(body)}")Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})+{:error, :request_failed}{:error, reason} ->log_ce_error("retrieving report for #{report_request.dataset}", reason)- Logger.debug(+ Logger.warning("[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}: #{inspect(reason)}")- {:error, :request_failed}+ Sentry.Context.set_extra_context(%{ga_response: %{body: inspect(reason), status: 0}})++ {:error, {:socket_failed, dataset: report_request.dataset, offset: report_request.offset}}endenddiff --git a/lib/plausible/imported/google_analytics4.ex b/lib/plausible/imported/google_analytics4.exindex 0b72c7a3f9..2118d57972 100644--- a/lib/plausible/imported/google_analytics4.ex+++ b/lib/plausible/imported/google_analytics4.ex@@ -8,6 +8,7 @@ defmodule Plausible.Imported.GoogleAnalytics4 doalias Plausible.Importedalias Plausible.Repo+ @recoverable_errors [:rate_limit_exceeded, :socket_failed, :server_failed]@missing_values ["(none)", "(not set)", "(not provided)", "(other)"]@impl true@@ -86,16 +87,18 @@ defmodule Plausible.Imported.GoogleAnalytics4 doendresume_opts = Keyword.take(opts, [:dataset, :offset])+ fetch_opts = Keyword.get(opts, :fetch_opts, [])try doresult =Plausible.Google.GA4.API.import_analytics(date_range, property, auth,persist_fn: persist_fn,+ fetch_opts: fetch_opts,resume_opts: resume_opts)case result do- {:error, {:rate_limit_exceeded, details}} ->+ {:error, {error, details}} when error in @recoverable_errors ->site_import = Repo.preload(site_import, [:site, :imported_by])dataset = Keyword.fetch!(details, :dataset)offset = Keyword.fetch!(details, :offset)@@ -121,7 +124,7 @@ defmodule Plausible.Imported.GoogleAnalytics4 doresume_import_opts)- {:error, :rate_limit_exceeded, skip_purge?: true, skip_mark_failed?: true}+ {:error, error, skip_purge?: true, skip_mark_failed?: true}other ->otherdiff --git a/test/plausible/imported/google_analytics4_test.exs b/test/plausible/imported/google_analytics4_test.exsindex 29624b8f28..2318402996 100644--- a/test/plausible/imported/google_analytics4_test.exs+++ b/test/plausible/imported/google_analytics4_test.exs@@ -4,6 +4,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doimport Moximport Ecto.Query, only: [from: 2]+ import ExUnit.CaptureLogalias Plausible.ClickhouseRepoalias Plausible.Repo@@ -336,6 +337,130 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert await_clickhouse_count(query, count)end)end++ @recoverable_errors [+ {+ Macro.escape(+ Plausible.HTTPClient.Non200Error.new(%Finch.Response{+ status: 500,+ body: "Internal server error"+ })+ ),+ :server_failed,+ ~s|Request failed for imported_sources with code 500: "Internal server error"|+ },+ {+ :timeout,+ :socket_failed,+ ~s|Request failed for imported_sources: :timeout|+ }+ ]++ for {error_mock, error_returned, log_message} <- @recoverable_errors do+ test "handles #{error_returned} gracefully", %{user: user, site: site} do+ future = DateTime.add(DateTime.utc_now(), 3600, :second)++ {:ok, job} =+ Plausible.Imported.GoogleAnalytics4.new_import(+ site,+ user,+ label: "properties/123456",+ property: "properties/123456",+ start_date: ~D[2024-01-01],+ end_date: ~D[2024-01-31],+ access_token: "redacted_access_token",+ refresh_token: "redacted_refresh_token",+ token_expires_at: DateTime.to_iso8601(future)+ )++ site_import = Plausible.Imported.get_import(site, job.args.import_id)++ opts = job |> Repo.reload!() |> Map.get(:args) |> GoogleAnalytics4.parse_args()++ opts =+ opts+ |> Keyword.put(:flush_interval_ms, 10)+ |> Keyword.put(:fetch_opts, max_attempts: 2, sleep_time: 50)++ expect(Plausible.HTTPClient.Mock, :post, fn _url, headers, _body, _opts ->+ assert [{"Authorization", "Bearer redacted_access_token"}] == headers+ {:ok, %Finch.Response{status: 200, body: List.first(@full_report_mock)}}+ end)++ expect(Plausible.HTTPClient.Mock, :post, 2, fn _url, headers, _body, _opts ->+ assert [{"Authorization", "Bearer redacted_access_token"}] == headers++ {:error, unquote(error_mock)}+ end)++ assert capture_log(fn ->+ assert {:error, unquote(error_returned),+ skip_purge?: true, skip_mark_failed?: true} =+ GoogleAnalytics4.import_data(site_import, opts)+ end) =~ unquote(log_message)++ in_65_minutes = DateTime.add(DateTime.utc_now(), 3900, :second)++ assert_enqueued(+ worker: Plausible.Workers.ImportAnalytics,+ args: %{resume_from_import_id: site_import.id},+ scheduled_at: {in_65_minutes, delta: 10}+ )++ [%{args: resume_args}, _] = all_enqueued()++ resume_opts = GoogleAnalytics4.parse_args(resume_args)+ resume_opts = Keyword.put(resume_opts, :flush_interval_ms, 10)+ site_import = Repo.reload!(site_import)++ Enum.each(Plausible.Imported.tables(), fn table ->+ count =+ case table do+ "imported_visitors" -> 31+ "imported_sources" -> 0+ "imported_pages" -> 0+ "imported_entry_pages" -> 0+ "imported_exit_pages" -> 0+ "imported_locations" -> 0+ "imported_devices" -> 0+ "imported_browsers" -> 0+ "imported_operating_systems" -> 0+ "imported_custom_events" -> 0+ end++ query = from(imported in table, where: imported.site_id == ^site.id)+ assert await_clickhouse_count(query, count)+ end)++ for report <- tl(@full_report_mock) do+ expect(Plausible.HTTPClient.Mock, :post, fn _url, headers, _body, _opts ->+ assert [{"Authorization", "Bearer redacted_access_token"}] == headers+ {:ok, %Finch.Response{status: 200, body: report}}+ end)+ end++ assert :ok = GoogleAnalytics4.import_data(site_import, resume_opts)++ Enum.each(Plausible.Imported.tables(), fn table ->+ count =+ case table do+ "imported_sources" -> 210+ "imported_visitors" -> 31+ "imported_pages" -> 3340+ "imported_entry_pages" -> 2934+ "imported_exit_pages" -> 0+ "imported_locations" -> 2291+ "imported_devices" -> 93+ "imported_browsers" -> 233+ "imported_operating_systems" -> 1068+ "imported_custom_events" -> 56+ end++ query = from(imported in table, where: imported.site_id == ^site.id)+ assert await_clickhouse_count(query, count)+ end)+ end+ endenddefp assert_active_visitors(site_import) do