Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Improve GA4 import error handling (#4714)

Adrian Gruntkowskibe83b7829df7
diff --git a/lib/plausible/google/ga4/api.ex b/lib/plausible/google/ga4/api.ex
index 64da14ba0f..7299149814 100644
--- a/lib/plausible/google/ga4/api.ex
+++ b/lib/plausible/google/ga4/api.ex
@@ -73,6 +73,7 @@ defmodule Plausible.Google.GA4.API do
def import_analytics(date_range, property, auth, opts) do
persist_fn = Keyword.fetch!(opts, :persist_fn)
+ fetch_opts = Keyword.get(opts, :fetch_opts, [])
resume_opts = Keyword.get(opts, :resume_opts, [])
Logger.debug(
@@ -80,11 +81,18 @@ defmodule Plausible.Google.GA4.API do
)
with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do
- do_import_analytics(date_range, property, access_token, persist_fn, resume_opts)
+ do_import_analytics(date_range, property, access_token, persist_fn, fetch_opts, resume_opts)
end
end
- defp do_import_analytics(date_range, property, access_token, persist_fn, [] = _resume_opts) do
+ defp do_import_analytics(
+ date_range,
+ property,
+ access_token,
+ persist_fn,
+ fetch_opts,
+ [] = _resume_opts
+ ) do
Enum.reduce_while(GA4.ReportRequest.full_report(), :ok, fn report_request, :ok ->
Logger.debug(
"[#{inspect(__MODULE__)}:#{property}] Starting to import #{report_request.dataset}"
@@ -92,14 +100,21 @@ defmodule Plausible.Google.GA4.API do
report_request = prepare_request(report_request, date_range, property, access_token)
- case fetch_and_persist(report_request, persist_fn: persist_fn) do
+ case fetch_and_persist(report_request, persist_fn: persist_fn, fetch_opts: fetch_opts) do
:ok -> {:cont, :ok}
{:error, _} = error -> {:halt, error}
end
end)
end
- defp do_import_analytics(date_range, property, access_token, persist_fn, resume_opts) do
+ defp do_import_analytics(
+ date_range,
+ property,
+ access_token,
+ persist_fn,
+ fetch_opts,
+ resume_opts
+ ) do
dataset = Keyword.fetch!(resume_opts, :dataset)
offset = Keyword.fetch!(resume_opts, :offset)
@@ -122,7 +137,7 @@ defmodule Plausible.Google.GA4.API do
|> prepare_request(date_range, property, access_token)
|> Map.put(:offset, request_offset)
- case fetch_and_persist(report_request, persist_fn: persist_fn) do
+ case fetch_and_persist(report_request, persist_fn: persist_fn, fetch_opts: fetch_opts) do
:ok -> {:cont, :ok}
{:error, _} = error -> {:halt, error}
end
@@ -134,7 +149,9 @@ defmodule Plausible.Google.GA4.API do
def fetch_and_persist(%GA4.ReportRequest{} = report_request, opts \\ []) do
persist_fn = Keyword.fetch!(opts, :persist_fn)
attempt = Keyword.get(opts, :attempt, 1)
- sleep_time = Keyword.get(opts, :sleep_time, @backoff_factor)
+ fetch_opts = Keyword.get(opts, :fetch_opts, [])
+ max_attempts = Keyword.get(fetch_opts, :max_attempts, @max_attempts)
+ sleep_time = Keyword.get(fetch_opts, :sleep_time, @backoff_factor)
case GA4.HTTP.get_report(report_request) do
{:ok, {_, 0}} ->
@@ -168,7 +185,7 @@ defmodule Plausible.Google.GA4.API do
{:error, {:rate_limit_exceeded, details}}
{:error, cause} ->
- if attempt >= @max_attempts do
+ if attempt >= max_attempts do
Logger.debug(
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Terminating."
)
diff --git a/lib/plausible/google/ga4/http.ex b/lib/plausible/google/ga4/http.ex
index 68533a917a..437fd1ed10 100644
--- a/lib/plausible/google/ga4/http.ex
+++ b/lib/plausible/google/ga4/http.ex
@@ -69,24 +69,38 @@ defmodule Plausible.Google.GA4.HTTP do
{:error,
{:rate_limit_exceeded, dataset: report_request.dataset, offset: report_request.offset}}
+ {:error, %{reason: %{status: status, body: body}} = error} when status >= 500 ->
+ log_ce_error("retrieving report for #{report_request.dataset}", error)
+
+ Logger.warning(
+ "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} with code #{status}: #{inspect(body)}"
+ )
+
+ Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
+
+ {:error, {:server_failed, dataset: report_request.dataset, offset: report_request.offset}}
+
{:error, %{reason: %{status: status, body: body}} = error} ->
log_ce_error("retrieving report for #{report_request.dataset}", error)
- Logger.debug(
+ Logger.warning(
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} with code #{status}: #{inspect(body)}"
)
Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
+
{:error, :request_failed}
{:error, reason} ->
log_ce_error("retrieving report for #{report_request.dataset}", reason)
- Logger.debug(
+ Logger.warning(
"[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}: #{inspect(reason)}"
)
- {:error, :request_failed}
+ Sentry.Context.set_extra_context(%{ga_response: %{body: inspect(reason), status: 0}})
+
+ {:error, {:socket_failed, dataset: report_request.dataset, offset: report_request.offset}}
end
end
diff --git a/lib/plausible/imported/google_analytics4.ex b/lib/plausible/imported/google_analytics4.ex
index 0b72c7a3f9..2118d57972 100644
--- a/lib/plausible/imported/google_analytics4.ex
+++ b/lib/plausible/imported/google_analytics4.ex
@@ -8,6 +8,7 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
alias Plausible.Imported
alias Plausible.Repo
+ @recoverable_errors [:rate_limit_exceeded, :socket_failed, :server_failed]
@missing_values ["(none)", "(not set)", "(not provided)", "(other)"]
@impl true
@@ -86,16 +87,18 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
end
resume_opts = Keyword.take(opts, [:dataset, :offset])
+ fetch_opts = Keyword.get(opts, :fetch_opts, [])
try do
result =
Plausible.Google.GA4.API.import_analytics(date_range, property, auth,
persist_fn: persist_fn,
+ fetch_opts: fetch_opts,
resume_opts: resume_opts
)
case result do
- {:error, {:rate_limit_exceeded, details}} ->
+ {:error, {error, details}} when error in @recoverable_errors ->
site_import = Repo.preload(site_import, [:site, :imported_by])
dataset = Keyword.fetch!(details, :dataset)
offset = Keyword.fetch!(details, :offset)
@@ -121,7 +124,7 @@ defmodule Plausible.Imported.GoogleAnalytics4 do
resume_import_opts
)
- {:error, :rate_limit_exceeded, skip_purge?: true, skip_mark_failed?: true}
+ {:error, error, skip_purge?: true, skip_mark_failed?: true}
other ->
other
diff --git a/test/plausible/imported/google_analytics4_test.exs b/test/plausible/imported/google_analytics4_test.exs
index 29624b8f28..2318402996 100644
--- a/test/plausible/imported/google_analytics4_test.exs
+++ b/test/plausible/imported/google_analytics4_test.exs
@@ -4,6 +4,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
import Mox
import Ecto.Query, only: [from: 2]
+ import ExUnit.CaptureLog
alias Plausible.ClickhouseRepo
alias Plausible.Repo
@@ -336,6 +337,130 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do
assert await_clickhouse_count(query, count)
end)
end
+
+ @recoverable_errors [
+ {
+ Macro.escape(
+ Plausible.HTTPClient.Non200Error.new(%Finch.Response{
+ status: 500,
+ body: "Internal server error"
+ })
+ ),
+ :server_failed,
+ ~s|Request failed for imported_sources with code 500: "Internal server error"|
+ },
+ {
+ :timeout,
+ :socket_failed,
+ ~s|Request failed for imported_sources: :timeout|
+ }
+ ]
+
+ for {error_mock, error_returned, log_message} <- @recoverable_errors do
+ test "handles #{error_returned} gracefully", %{user: user, site: site} do
+ future = DateTime.add(DateTime.utc_now(), 3600, :second)
+
+ {:ok, job} =
+ Plausible.Imported.GoogleAnalytics4.new_import(
+ site,
+ user,
+ label: "properties/123456",
+ property: "properties/123456",
+ start_date: ~D[2024-01-01],
+ end_date: ~D[2024-01-31],
+ access_token: "redacted_access_token",
+ refresh_token: "redacted_refresh_token",
+ token_expires_at: DateTime.to_iso8601(future)
+ )
+
+ site_import = Plausible.Imported.get_import(site, job.args.import_id)
+
+ opts = job |> Repo.reload!() |> Map.get(:args) |> GoogleAnalytics4.parse_args()
+
+ opts =
+ opts
+ |> Keyword.put(:flush_interval_ms, 10)
+ |> Keyword.put(:fetch_opts, max_attempts: 2, sleep_time: 50)
+
+ expect(Plausible.HTTPClient.Mock, :post, fn _url, headers, _body, _opts ->
+ assert [{"Authorization", "Bearer redacted_access_token"}] == headers
+ {:ok, %Finch.Response{status: 200, body: List.first(@full_report_mock)}}
+ end)
+
+ expect(Plausible.HTTPClient.Mock, :post, 2, fn _url, headers, _body, _opts ->
+ assert [{"Authorization", "Bearer redacted_access_token"}] == headers
+
+ {:error, unquote(error_mock)}
+ end)
+
+ assert capture_log(fn ->
+ assert {:error, unquote(error_returned),
+ skip_purge?: true, skip_mark_failed?: true} =
+ GoogleAnalytics4.import_data(site_import, opts)
+ end) =~ unquote(log_message)
+
+ in_65_minutes = DateTime.add(DateTime.utc_now(), 3900, :second)
+
+ assert_enqueued(
+ worker: Plausible.Workers.ImportAnalytics,
+ args: %{resume_from_import_id: site_import.id},
+ scheduled_at: {in_65_minutes, delta: 10}
+ )
+
+ [%{args: resume_args}, _] = all_enqueued()
+
+ resume_opts = GoogleAnalytics4.parse_args(resume_args)
+ resume_opts = Keyword.put(resume_opts, :flush_interval_ms, 10)
+ site_import = Repo.reload!(site_import)
+
+ Enum.each(Plausible.Imported.tables(), fn table ->
+ count =
+ case table do
+ "imported_visitors" -> 31
+ "imported_sources" -> 0
+ "imported_pages" -> 0
+ "imported_entry_pages" -> 0
+ "imported_exit_pages" -> 0
+ "imported_locations" -> 0
+ "imported_devices" -> 0
+ "imported_browsers" -> 0
+ "imported_operating_systems" -> 0
+ "imported_custom_events" -> 0
+ end
+
+ query = from(imported in table, where: imported.site_id == ^site.id)
+ assert await_clickhouse_count(query, count)
+ end)
+
+ for report <- tl(@full_report_mock) do
+ expect(Plausible.HTTPClient.Mock, :post, fn _url, headers, _body, _opts ->
+ assert [{"Authorization", "Bearer redacted_access_token"}] == headers
+ {:ok, %Finch.Response{status: 200, body: report}}
+ end)
+ end
+
+ assert :ok = GoogleAnalytics4.import_data(site_import, resume_opts)
+
+ Enum.each(Plausible.Imported.tables(), fn table ->
+ count =
+ case table do
+ "imported_sources" -> 210
+ "imported_visitors" -> 31
+ "imported_pages" -> 3340
+ "imported_entry_pages" -> 2934
+ "imported_exit_pages" -> 0
+ "imported_locations" -> 2291
+ "imported_devices" -> 93
+ "imported_browsers" -> 233
+ "imported_operating_systems" -> 1068
+ "imported_custom_events" -> 56
+ end
+
+ query = from(imported in table, where: imported.site_id == ^site.id)
+ assert await_clickhouse_count(query, count)
+ end)
+ end
+ end
end
defp assert_active_visitors(site_import) do