Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Increase Google Analytics timeout and lower page size (#2326)
Vinicius Brasilfb05e447f963
diff --git a/fixture/vcr_cassettes/google_analytics_import#1.json b/fixture/vcr_cassettes/google_analytics_import#1.jsonindex 012a91cf86..eb48291847 100644--- a/fixture/vcr_cassettes/google_analytics_import#1.json+++ b/fixture/vcr_cassettes/google_analytics_import#1.json@@ -1,7 +1,7 @@[{"request": {- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:hostname\"},{\"histogramBuckets\":[],\"name\":\"ga:pagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:pageviews\"},{\"expression\":\"ga:exits\"},{\"expression\":\"ga:timeOnPage\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:hostname\"},{\"histogramBuckets\":[],\"name\":\"ga:pagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:pageviews\"},{\"expression\":\"ga:exits\"},{\"expression\":\"ga:timeOnPage\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}","headers": {"Authorization": "***"},@@ -32,7 +32,7 @@},{"request": {- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:pageviews\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:pageviews\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}","headers": {"Authorization": "***"},@@ -63,7 +63,7 @@},{"request": {- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:source\"},{\"histogramBuckets\":[],\"name\":\"ga:medium\"},{\"histogramBuckets\":[],\"name\":\"ga:campaign\"},{\"histogramBuckets\":[],\"name\":\"ga:adContent\"},{\"histogramBuckets\":[],\"name\":\"ga:keyword\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:source\"},{\"histogramBuckets\":[],\"name\":\"ga:medium\"},{\"histogramBuckets\":[],\"name\":\"ga:campaign\"},{\"histogramBuckets\":[],\"name\":\"ga:adContent\"},{\"histogramBuckets\":[],\"name\":\"ga:keyword\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}","headers": {"Authorization": "***"},@@ -94,7 +94,7 @@},{"request": {- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:landingPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:entrances\"},{\"expression\":\"ga:sessionDuration\"},{\"expression\":\"ga:bounces\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:landingPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:entrances\"},{\"expression\":\"ga:sessionDuration\"},{\"expression\":\"ga:bounces\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}","headers": {"Authorization": "***"},@@ -125,7 +125,7 @@},{"request": {- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:hostname\"},{\"histogramBuckets\":[],\"name\":\"ga:pagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:pageviews\"},{\"expression\":\"ga:exits\"},{\"expression\":\"ga:timeOnPage\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":\"10000\",\"viewId\":\"54297898\"}]}",+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:hostname\"},{\"histogramBuckets\":[],\"name\":\"ga:pagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:pageviews\"},{\"expression\":\"ga:exits\"},{\"expression\":\"ga:timeOnPage\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":\"10000\",\"viewId\":\"54297898\"}]}","headers": {"Authorization": "***"},@@ -156,7 +156,7 @@},{"request": {- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:landingPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:entrances\"},{\"expression\":\"ga:sessionDuration\"},{\"expression\":\"ga:bounces\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":\"10000\",\"viewId\":\"54297898\"}]}",+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:landingPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:entrances\"},{\"expression\":\"ga:sessionDuration\"},{\"expression\":\"ga:bounces\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":\"10000\",\"viewId\":\"54297898\"}]}","headers": {"Authorization": "***"},@@ -187,7 +187,7 @@},{"request": {- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:deviceCategory\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:deviceCategory\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}","headers": {"Authorization": "***"},@@ -218,7 +218,7 @@},{"request": {- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:exitPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:exits\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:exitPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:exits\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}","headers": {"Authorization": "***"},@@ -249,7 +249,7 @@},{"request": {- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:countryIsoCode\"},{\"histogramBuckets\":[],\"name\":\"ga:regionIsoCode\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:countryIsoCode\"},{\"histogramBuckets\":[],\"name\":\"ga:regionIsoCode\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}","headers": {"Authorization": "***"},@@ -280,7 +280,7 @@},{"request": {- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:browser\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:browser\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}","headers": {"Authorization": "***"},@@ -311,7 +311,7 @@},{"request": {- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:exitPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:exits\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":\"10000\",\"viewId\":\"54297898\"}]}",+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:exitPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:exits\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":\"10000\",\"viewId\":\"54297898\"}]}","headers": {"Authorization": "***"},@@ -342,7 +342,7 @@},{"request": {- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:operatingSystem\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:operatingSystem\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}","headers": {"Authorization": "***"},diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.exindex f0c91ccfb7..2e58ccd794 100644--- a/lib/plausible/google/api.ex+++ b/lib/plausible/google/api.ex@@ -114,7 +114,7 @@ defmodule Plausible.Google.Api doexpires_at :: String.t()}- @per_page 10_000+ @per_page 7_500@max_attempts 5@spec import_analytics(Plausible.Site.t(), Date.Range.t(), String.t(), import_auth()) :::ok | {:error, term()}diff --git a/lib/plausible/google/http.ex b/lib/plausible/google/http.exindex e8957f0a27..372fd19126 100644--- a/lib/plausible/google/http.ex+++ b/lib/plausible/google/http.ex@@ -34,7 +34,7 @@ defmodule Plausible.Google.HTTP do[{"Authorization", "Bearer #{report_request.access_token}"}],params)- |> http_client.request(Plausible.Finch, receive_timeout: 30_000)+ |> http_client.request(Plausible.Finch, receive_timeout: 60_000)with {:ok, %{status: 200, body: body}} <- response,{:ok, report} <- parse_report_from_response(body),diff --git a/test/plausible/google/api_test.exs b/test/plausible/google/api_test.exsindex 0002befc45..01c86aede2 100644--- a/test/plausible/google/api_test.exs+++ b/test/plausible/google/api_test.exs@@ -77,11 +77,11 @@ defmodule Plausible.Google.ApiTest dobuffer: buffer)- assert_receive({Finch, :request, [_, _, [receive_timeout: 30_000]]})- assert_receive({Finch, :request, [_, _, [receive_timeout: 30_000]]})- assert_receive({Finch, :request, [_, _, [receive_timeout: 30_000]]})- assert_receive({Finch, :request, [_, _, [receive_timeout: 30_000]]})- assert_receive({Finch, :request, [_, _, [receive_timeout: 30_000]]})+ assert_receive({Finch, :request, [_, _, [receive_timeout: 60_000]]})+ assert_receive({Finch, :request, [_, _, [receive_timeout: 60_000]]})+ assert_receive({Finch, :request, [_, _, [receive_timeout: 60_000]]})+ assert_receive({Finch, :request, [_, _, [receive_timeout: 60_000]]})+ assert_receive({Finch, :request, [_, _, [receive_timeout: 60_000]]})endtest "does not fail when report does not have rows key", %{site: site, buffer: buffer} do