Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Increase Google Analytics timeout and lower page size (#2326)

Vinicius Brasilfb05e447f963
diff --git a/fixture/vcr_cassettes/google_analytics_import#1.json b/fixture/vcr_cassettes/google_analytics_import#1.json
index 012a91cf86..eb48291847 100644
--- a/fixture/vcr_cassettes/google_analytics_import#1.json
+++ b/fixture/vcr_cassettes/google_analytics_import#1.json
@@ -1,7 +1,7 @@
[
{
"request": {
- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:hostname\"},{\"histogramBuckets\":[],\"name\":\"ga:pagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:pageviews\"},{\"expression\":\"ga:exits\"},{\"expression\":\"ga:timeOnPage\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:hostname\"},{\"histogramBuckets\":[],\"name\":\"ga:pagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:pageviews\"},{\"expression\":\"ga:exits\"},{\"expression\":\"ga:timeOnPage\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@@ -32,7 +32,7 @@
},
{
"request": {
- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:pageviews\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:pageviews\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@@ -63,7 +63,7 @@
},
{
"request": {
- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:source\"},{\"histogramBuckets\":[],\"name\":\"ga:medium\"},{\"histogramBuckets\":[],\"name\":\"ga:campaign\"},{\"histogramBuckets\":[],\"name\":\"ga:adContent\"},{\"histogramBuckets\":[],\"name\":\"ga:keyword\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:source\"},{\"histogramBuckets\":[],\"name\":\"ga:medium\"},{\"histogramBuckets\":[],\"name\":\"ga:campaign\"},{\"histogramBuckets\":[],\"name\":\"ga:adContent\"},{\"histogramBuckets\":[],\"name\":\"ga:keyword\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@@ -94,7 +94,7 @@
},
{
"request": {
- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:landingPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:entrances\"},{\"expression\":\"ga:sessionDuration\"},{\"expression\":\"ga:bounces\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:landingPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:entrances\"},{\"expression\":\"ga:sessionDuration\"},{\"expression\":\"ga:bounces\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@@ -125,7 +125,7 @@
},
{
"request": {
- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:hostname\"},{\"histogramBuckets\":[],\"name\":\"ga:pagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:pageviews\"},{\"expression\":\"ga:exits\"},{\"expression\":\"ga:timeOnPage\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":\"10000\",\"viewId\":\"54297898\"}]}",
+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:hostname\"},{\"histogramBuckets\":[],\"name\":\"ga:pagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:pageviews\"},{\"expression\":\"ga:exits\"},{\"expression\":\"ga:timeOnPage\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":\"10000\",\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@@ -156,7 +156,7 @@
},
{
"request": {
- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:landingPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:entrances\"},{\"expression\":\"ga:sessionDuration\"},{\"expression\":\"ga:bounces\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":\"10000\",\"viewId\":\"54297898\"}]}",
+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:landingPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:entrances\"},{\"expression\":\"ga:sessionDuration\"},{\"expression\":\"ga:bounces\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":\"10000\",\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@@ -187,7 +187,7 @@
},
{
"request": {
- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:deviceCategory\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:deviceCategory\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@@ -218,7 +218,7 @@
},
{
"request": {
- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:exitPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:exits\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:exitPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:exits\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@@ -249,7 +249,7 @@
},
{
"request": {
- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:countryIsoCode\"},{\"histogramBuckets\":[],\"name\":\"ga:regionIsoCode\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:countryIsoCode\"},{\"histogramBuckets\":[],\"name\":\"ga:regionIsoCode\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@@ -280,7 +280,7 @@
},
{
"request": {
- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:browser\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:browser\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@@ -311,7 +311,7 @@
},
{
"request": {
- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:exitPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:exits\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":\"10000\",\"viewId\":\"54297898\"}]}",
+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:exitPagePath\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:exits\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":\"10000\",\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
@@ -342,7 +342,7 @@
},
{
"request": {
- "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:operatingSystem\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":10000,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
+ "body": "{\"reportRequests\":[{\"dateRanges\":[{\"endDate\":\"2022-07-19\",\"startDate\":\"2011-01-01\"}],\"dimensions\":[{\"histogramBuckets\":[],\"name\":\"ga:date\"},{\"histogramBuckets\":[],\"name\":\"ga:operatingSystem\"}],\"hideTotals\":true,\"hideValueRanges\":true,\"metrics\":[{\"expression\":\"ga:users\"},{\"expression\":\"ga:sessions\"},{\"expression\":\"ga:bounces\"},{\"expression\":\"ga:sessionDuration\"}],\"orderBys\":[{\"fieldName\":\"ga:date\",\"sortOrder\":\"DESCENDING\"}],\"pageSize\":7500,\"pageToken\":null,\"viewId\":\"54297898\"}]}",
"headers": {
"Authorization": "***"
},
diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex
index f0c91ccfb7..2e58ccd794 100644
--- a/lib/plausible/google/api.ex
+++ b/lib/plausible/google/api.ex
@@ -114,7 +114,7 @@ defmodule Plausible.Google.Api do
expires_at :: String.t()
}
- @per_page 10_000
+ @per_page 7_500
@max_attempts 5
@spec import_analytics(Plausible.Site.t(), Date.Range.t(), String.t(), import_auth()) ::
:ok | {:error, term()}
diff --git a/lib/plausible/google/http.ex b/lib/plausible/google/http.ex
index e8957f0a27..372fd19126 100644
--- a/lib/plausible/google/http.ex
+++ b/lib/plausible/google/http.ex
@@ -34,7 +34,7 @@ defmodule Plausible.Google.HTTP do
[{"Authorization", "Bearer #{report_request.access_token}"}],
params
)
- |> http_client.request(Plausible.Finch, receive_timeout: 30_000)
+ |> http_client.request(Plausible.Finch, receive_timeout: 60_000)
with {:ok, %{status: 200, body: body}} <- response,
{:ok, report} <- parse_report_from_response(body),
diff --git a/test/plausible/google/api_test.exs b/test/plausible/google/api_test.exs
index 0002befc45..01c86aede2 100644
--- a/test/plausible/google/api_test.exs
+++ b/test/plausible/google/api_test.exs
@@ -77,11 +77,11 @@ defmodule Plausible.Google.ApiTest do
buffer: buffer
)
- assert_receive({Finch, :request, [_, _, [receive_timeout: 30_000]]})
- assert_receive({Finch, :request, [_, _, [receive_timeout: 30_000]]})
- assert_receive({Finch, :request, [_, _, [receive_timeout: 30_000]]})
- assert_receive({Finch, :request, [_, _, [receive_timeout: 30_000]]})
- assert_receive({Finch, :request, [_, _, [receive_timeout: 30_000]]})
+ assert_receive({Finch, :request, [_, _, [receive_timeout: 60_000]]})
+ assert_receive({Finch, :request, [_, _, [receive_timeout: 60_000]]})
+ assert_receive({Finch, :request, [_, _, [receive_timeout: 60_000]]})
+ assert_receive({Finch, :request, [_, _, [receive_timeout: 60_000]]})
+ assert_receive({Finch, :request, [_, _, [receive_timeout: 60_000]]})
end
test "does not fail when report does not have rows key", %{site: site, buffer: buffer} do