Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Fix Google Search Console API call (#2090)

Vinicius Brasil6b8ed125676a
diff --git a/fixture/vcr_cassettes/google_analytics_stats#with_page.json b/fixture/vcr_cassettes/google_analytics_stats#with_page.json
new file mode 100644
index 0000000000..a2298fd535
--- /dev/null
+++ b/fixture/vcr_cassettes/google_analytics_stats#with_page.json
@@ -0,0 +1,33 @@
+[
+ {
+ "request": {
+ "body": "{\"dimensionFilterGroups\":[{\"filters\":[{\"dimension\":\"page\",\"expression\":\"https://sc-domain%3Adummy.test5\"}]}],\"dimensions\":[\"query\"],\"endDate\":\"2022-01-05\",\"rowLimit\":5,\"startDate\":\"2022-01-01\"}",
+ "headers": {
+ "Authorization": "Bearer 123"
+ },
+ "method": "post",
+ "options": [],
+ "request_body": "",
+ "url": "https://www.googleapis.com/webmasters/v3/sites/sc-domain%3Adummy.test/searchAnalytics/query"
+ },
+ "response": {
+ "binary": false,
+ "body": "{\"rows\":[{\"keys\":[\"keyword1\",\"keyword2\"],\"clicks\":25.0,\"impressions\":50.0,\"ctr\":0.3,\"position\":2.0},{\"keys\":[\"keyword3\",\"keyword4\"],\"clicks\":15.0,\"impressions\":25.0,\"ctr\":0.5,\"position\":4.0}],\"responseAggregationType\":\"auto\"}",
+ "headers": {
+ "vary": "X-Origin",
+ "content-type": "application/json; charset=UTF-8",
+ "date": "Wed, 10 Aug 2022 14:55:07 GMT",
+ "server": "ESF",
+ "cache-control": "private",
+ "x-xss-protection": "0",
+ "x-frame-options": "SAMEORIGIN",
+ "x-content-type-options": "nosniff",
+ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"",
+ "accept-ranges": "none",
+ "transfer-encoding": "chunked"
+ },
+ "status_code": 200,
+ "type": "ok"
+ }
+ }
+]
diff --git a/fixture/vcr_cassettes/google_analytics_stats#without_page.json b/fixture/vcr_cassettes/google_analytics_stats#without_page.json
new file mode 100644
index 0000000000..1bdafb3859
--- /dev/null
+++ b/fixture/vcr_cassettes/google_analytics_stats#without_page.json
@@ -0,0 +1,33 @@
+[
+ {
+ "request": {
+ "body": "{\"dimensionFilterGroups\":{},\"dimensions\":[\"query\"],\"endDate\":\"2022-01-05\",\"rowLimit\":5,\"startDate\":\"2022-01-01\"}",
+ "headers": {
+ "Authorization": "Bearer 123"
+ },
+ "method": "post",
+ "options": [],
+ "request_body": "",
+ "url": "https://www.googleapis.com/webmasters/v3/sites/sc-domain%3Adummy.test/searchAnalytics/query"
+ },
+ "response": {
+ "binary": false,
+ "body": "{\"rows\":[{\"keys\":[\"keyword1\",\"keyword2\"],\"clicks\":25.0,\"impressions\":50.0,\"ctr\":0.3,\"position\":2.0},{\"keys\":[\"keyword3\",\"keyword4\"],\"clicks\":15.0,\"impressions\":25.0,\"ctr\":0.5,\"position\":4.0}],\"responseAggregationType\":\"auto\"}",
+ "headers": {
+ "vary": "X-Origin",
+ "content-type": "application/json; charset=UTF-8",
+ "date": "Wed, 10 Aug 2022 14:55:07 GMT",
+ "server": "ESF",
+ "cache-control": "private",
+ "x-xss-protection": "0",
+ "x-frame-options": "SAMEORIGIN",
+ "x-content-type-options": "nosniff",
+ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"",
+ "accept-ranges": "none",
+ "transfer-encoding": "chunked"
+ },
+ "status_code": 200,
+ "type": "ok"
+ }
+ }
+]
diff --git a/fixture/vcr_cassettes/google_analytics_stats.json b/fixture/vcr_cassettes/google_analytics_stats.json
new file mode 100644
index 0000000000..1bdafb3859
--- /dev/null
+++ b/fixture/vcr_cassettes/google_analytics_stats.json
@@ -0,0 +1,33 @@
+[
+ {
+ "request": {
+ "body": "{\"dimensionFilterGroups\":{},\"dimensions\":[\"query\"],\"endDate\":\"2022-01-05\",\"rowLimit\":5,\"startDate\":\"2022-01-01\"}",
+ "headers": {
+ "Authorization": "Bearer 123"
+ },
+ "method": "post",
+ "options": [],
+ "request_body": "",
+ "url": "https://www.googleapis.com/webmasters/v3/sites/sc-domain%3Adummy.test/searchAnalytics/query"
+ },
+ "response": {
+ "binary": false,
+ "body": "{\"rows\":[{\"keys\":[\"keyword1\",\"keyword2\"],\"clicks\":25.0,\"impressions\":50.0,\"ctr\":0.3,\"position\":2.0},{\"keys\":[\"keyword3\",\"keyword4\"],\"clicks\":15.0,\"impressions\":25.0,\"ctr\":0.5,\"position\":4.0}],\"responseAggregationType\":\"auto\"}",
+ "headers": {
+ "vary": "X-Origin",
+ "content-type": "application/json; charset=UTF-8",
+ "date": "Wed, 10 Aug 2022 14:55:07 GMT",
+ "server": "ESF",
+ "cache-control": "private",
+ "x-xss-protection": "0",
+ "x-frame-options": "SAMEORIGIN",
+ "x-content-type-options": "nosniff",
+ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"",
+ "accept-ranges": "none",
+ "transfer-encoding": "chunked"
+ },
+ "status_code": 200,
+ "type": "ok"
+ }
+ }
+]
diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex
index 6bd149770d..8c3ac11d39 100644
--- a/lib/plausible/google/api.ex
+++ b/lib/plausible/google/api.ex
@@ -39,14 +39,17 @@ defmodule Plausible.Google.Api do
end
end
- def fetch_stats(site, %{date_range: date_range, filters: %{"page" => page}}, limit) do
- with {:ok, %{access_token: access_token, property: property}} <-
+ def fetch_stats(site, %{filters: %{} = filters, date_range: date_range}, limit) do
+ with site <- Plausible.Repo.preload(site, :google_auth),
+ {:ok, %{access_token: access_token, property: property}} <-
refresh_if_needed(site.google_auth),
- {:ok, stats} <- HTTP.list_stats(access_token, property, date_range, limit, page) do
+ {:ok, stats} <-
+ HTTP.list_stats(access_token, property, date_range, limit, filters["page"]) do
stats
|> Map.get("rows", [])
|> Enum.filter(fn row -> row["clicks"] > 0 end)
|> Enum.map(fn row -> %{name: row["keys"], visitors: round(row["clicks"])} end)
+ |> then(&{:ok, &1})
end
end
diff --git a/lib/plausible/google/http.ex b/lib/plausible/google/http.ex
index 46995bbc29..f9513e8198 100644
--- a/lib/plausible/google/http.ex
+++ b/lib/plausible/google/http.ex
@@ -108,25 +108,27 @@ defmodule Plausible.Google.HTTP do
dimensionFilterGroups: filter_groups
})
- "https://www.googleapis.com/webmasters/v3/sites/#{property}/searchAnalytics/query"
- |> HTTPoison.post!(params,
- "Content-Type": "application/json",
- Authorization: "Bearer #{access_token}"
+ :post
+ |> Finch.build(
+ "https://www.googleapis.com/webmasters/v3/sites/#{property}/searchAnalytics/query",
+ [{"Authorization", "Bearer #{access_token}"}],
+ params
)
+ |> Finch.request(Plausible.Finch)
|> case do
- %{status_code: 200, body: body} ->
+ {:ok, %{status: 200, body: body}} ->
{:ok, Jason.decode!(body)}
- %{status_code: 401, body: body} ->
+ {:ok, %{status: 401, body: body}} ->
Sentry.capture_message("Error fetching Google queries", extra: Jason.decode!(body))
{:error, :invalid_credentials}
- %{status_code: 403, body: body} ->
+ {:ok, %{status: 403, body: body}} ->
body = Jason.decode!(body)
Sentry.capture_message("Error fetching Google queries", extra: body)
{:error, get_in(body, ["error", "message"])}
- %{body: body} ->
+ {:ok, %{body: body}} ->
Sentry.capture_message("Error fetching Google queries", extra: Jason.decode!(body))
{:error, :unknown}
end
diff --git a/test/plausible/google/api_test.exs b/test/plausible/google/api_test.exs
index 47f5c09c05..d4ee6ccbab 100644
--- a/test/plausible/google/api_test.exs
+++ b/test/plausible/google/api_test.exs
@@ -1,18 +1,19 @@
defmodule Plausible.Google.ApiTest do
use Plausible.DataCase, async: true
+ use ExVCR.Mock, adapter: ExVCR.Adapter.Finch
alias Plausible.Google.Api
import Plausible.TestUtils
import Double
- @ok_response File.read!("fixture/ga_batch_report.json")
-
- def start_buffer(_setup_args) do
- {:ok, pid} = Plausible.Google.Buffer.start_link()
- {:ok, buffer: pid}
- end
+ setup [:create_user, :create_new_site]
describe "fetch_and_persist/4" do
- setup [:create_user, :create_new_site, :start_buffer]
+ @ok_response File.read!("fixture/ga_batch_report.json")
+
+ setup do
+ {:ok, pid} = Plausible.Google.Buffer.start_link()
+ {:ok, buffer: pid}
+ end
test "will fetch and persist import data from Google Analytics", %{site: site, buffer: buffer} do
finch_double =
@@ -84,4 +85,66 @@ defmodule Plausible.Google.ApiTest do
assert_receive({Finch, :request, [_, _]})
end
end
+
+ describe "fetch_stats/3" do
+ test "returns name and visitor count", %{user: user, site: site} do
+ use_cassette "google_analytics_stats", match_requests_on: [:request_body] do
+ insert(:google_auth,
+ user: user,
+ site: site,
+ property: "sc-domain:dummy.test",
+ expires: NaiveDateTime.add(NaiveDateTime.utc_now(), 3600)
+ )
+
+ query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}
+
+ assert {:ok,
+ [
+ %{name: ["keyword1", "keyword2"], visitors: 25},
+ %{name: ["keyword3", "keyword4"], visitors: 15}
+ ]} = Plausible.Google.Api.fetch_stats(site, query, 5)
+ end
+ end
+
+ test "returns next page when page argument is set", %{user: user, site: site} do
+ use_cassette "google_analytics_stats#with_page", match_requests_on: [:request_body] do
+ insert(:google_auth,
+ user: user,
+ site: site,
+ property: "sc-domain:dummy.test",
+ expires: NaiveDateTime.add(NaiveDateTime.utc_now(), 3600)
+ )
+
+ query = %Plausible.Stats.Query{
+ filters: %{"page" => 5},
+ date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])
+ }
+
+ assert {:ok,
+ [
+ %{name: ["keyword1", "keyword2"], visitors: 25},
+ %{name: ["keyword3", "keyword4"], visitors: 15}
+ ]} = Plausible.Google.Api.fetch_stats(site, query, 5)
+ end
+ end
+
+ test "defaults first page when page argument is not set", %{user: user, site: site} do
+ use_cassette "google_analytics_stats#without_page", match_requests_on: [:request_body] do
+ insert(:google_auth,
+ user: user,
+ site: site,
+ property: "sc-domain:dummy.test",
+ expires: NaiveDateTime.add(NaiveDateTime.utc_now(), 3600)
+ )
+
+ query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}
+
+ assert {:ok,
+ [
+ %{name: ["keyword1", "keyword2"], visitors: 25},
+ %{name: ["keyword3", "keyword4"], visitors: 15}
+ ]} = Plausible.Google.Api.fetch_stats(site, query, 5)
+ end
+ end
+ end
end