Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Fix Google Search Console API call (#2090)
Vinicius Brasil6b8ed125676a
diff --git a/fixture/vcr_cassettes/google_analytics_stats#with_page.json b/fixture/vcr_cassettes/google_analytics_stats#with_page.jsonnew file mode 100644index 0000000000..a2298fd535--- /dev/null+++ b/fixture/vcr_cassettes/google_analytics_stats#with_page.json@@ -0,0 +1,33 @@+[+ {+ "request": {+ "body": "{\"dimensionFilterGroups\":[{\"filters\":[{\"dimension\":\"page\",\"expression\":\"https://sc-domain%3Adummy.test5\"}]}],\"dimensions\":[\"query\"],\"endDate\":\"2022-01-05\",\"rowLimit\":5,\"startDate\":\"2022-01-01\"}",+ "headers": {+ "Authorization": "Bearer 123"+ },+ "method": "post",+ "options": [],+ "request_body": "",+ "url": "https://www.googleapis.com/webmasters/v3/sites/sc-domain%3Adummy.test/searchAnalytics/query"+ },+ "response": {+ "binary": false,+ "body": "{\"rows\":[{\"keys\":[\"keyword1\",\"keyword2\"],\"clicks\":25.0,\"impressions\":50.0,\"ctr\":0.3,\"position\":2.0},{\"keys\":[\"keyword3\",\"keyword4\"],\"clicks\":15.0,\"impressions\":25.0,\"ctr\":0.5,\"position\":4.0}],\"responseAggregationType\":\"auto\"}",+ "headers": {+ "vary": "X-Origin",+ "content-type": "application/json; charset=UTF-8",+ "date": "Wed, 10 Aug 2022 14:55:07 GMT",+ "server": "ESF",+ "cache-control": "private",+ "x-xss-protection": "0",+ "x-frame-options": "SAMEORIGIN",+ "x-content-type-options": "nosniff",+ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"",+ "accept-ranges": "none",+ "transfer-encoding": "chunked"+ },+ "status_code": 200,+ "type": "ok"+ }+ }+]diff --git a/fixture/vcr_cassettes/google_analytics_stats#without_page.json b/fixture/vcr_cassettes/google_analytics_stats#without_page.jsonnew file mode 100644index 0000000000..1bdafb3859--- /dev/null+++ b/fixture/vcr_cassettes/google_analytics_stats#without_page.json@@ -0,0 +1,33 @@+[+ {+ "request": {+ "body": "{\"dimensionFilterGroups\":{},\"dimensions\":[\"query\"],\"endDate\":\"2022-01-05\",\"rowLimit\":5,\"startDate\":\"2022-01-01\"}",+ "headers": {+ "Authorization": "Bearer 123"+ },+ "method": "post",+ "options": [],+ "request_body": "",+ "url": "https://www.googleapis.com/webmasters/v3/sites/sc-domain%3Adummy.test/searchAnalytics/query"+ },+ "response": {+ "binary": false,+ "body": "{\"rows\":[{\"keys\":[\"keyword1\",\"keyword2\"],\"clicks\":25.0,\"impressions\":50.0,\"ctr\":0.3,\"position\":2.0},{\"keys\":[\"keyword3\",\"keyword4\"],\"clicks\":15.0,\"impressions\":25.0,\"ctr\":0.5,\"position\":4.0}],\"responseAggregationType\":\"auto\"}",+ "headers": {+ "vary": "X-Origin",+ "content-type": "application/json; charset=UTF-8",+ "date": "Wed, 10 Aug 2022 14:55:07 GMT",+ "server": "ESF",+ "cache-control": "private",+ "x-xss-protection": "0",+ "x-frame-options": "SAMEORIGIN",+ "x-content-type-options": "nosniff",+ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"",+ "accept-ranges": "none",+ "transfer-encoding": "chunked"+ },+ "status_code": 200,+ "type": "ok"+ }+ }+]diff --git a/fixture/vcr_cassettes/google_analytics_stats.json b/fixture/vcr_cassettes/google_analytics_stats.jsonnew file mode 100644index 0000000000..1bdafb3859--- /dev/null+++ b/fixture/vcr_cassettes/google_analytics_stats.json@@ -0,0 +1,33 @@+[+ {+ "request": {+ "body": "{\"dimensionFilterGroups\":{},\"dimensions\":[\"query\"],\"endDate\":\"2022-01-05\",\"rowLimit\":5,\"startDate\":\"2022-01-01\"}",+ "headers": {+ "Authorization": "Bearer 123"+ },+ "method": "post",+ "options": [],+ "request_body": "",+ "url": "https://www.googleapis.com/webmasters/v3/sites/sc-domain%3Adummy.test/searchAnalytics/query"+ },+ "response": {+ "binary": false,+ "body": "{\"rows\":[{\"keys\":[\"keyword1\",\"keyword2\"],\"clicks\":25.0,\"impressions\":50.0,\"ctr\":0.3,\"position\":2.0},{\"keys\":[\"keyword3\",\"keyword4\"],\"clicks\":15.0,\"impressions\":25.0,\"ctr\":0.5,\"position\":4.0}],\"responseAggregationType\":\"auto\"}",+ "headers": {+ "vary": "X-Origin",+ "content-type": "application/json; charset=UTF-8",+ "date": "Wed, 10 Aug 2022 14:55:07 GMT",+ "server": "ESF",+ "cache-control": "private",+ "x-xss-protection": "0",+ "x-frame-options": "SAMEORIGIN",+ "x-content-type-options": "nosniff",+ "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\"",+ "accept-ranges": "none",+ "transfer-encoding": "chunked"+ },+ "status_code": 200,+ "type": "ok"+ }+ }+]diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.exindex 6bd149770d..8c3ac11d39 100644--- a/lib/plausible/google/api.ex+++ b/lib/plausible/google/api.ex@@ -39,14 +39,17 @@ defmodule Plausible.Google.Api doendend- def fetch_stats(site, %{date_range: date_range, filters: %{"page" => page}}, limit) do- with {:ok, %{access_token: access_token, property: property}} <-+ def fetch_stats(site, %{filters: %{} = filters, date_range: date_range}, limit) do+ with site <- Plausible.Repo.preload(site, :google_auth),+ {:ok, %{access_token: access_token, property: property}} <-refresh_if_needed(site.google_auth),- {:ok, stats} <- HTTP.list_stats(access_token, property, date_range, limit, page) do+ {:ok, stats} <-+ HTTP.list_stats(access_token, property, date_range, limit, filters["page"]) dostats|> Map.get("rows", [])|> Enum.filter(fn row -> row["clicks"] > 0 end)|> Enum.map(fn row -> %{name: row["keys"], visitors: round(row["clicks"])} end)+ |> then(&{:ok, &1})endenddiff --git a/lib/plausible/google/http.ex b/lib/plausible/google/http.exindex 46995bbc29..f9513e8198 100644--- a/lib/plausible/google/http.ex+++ b/lib/plausible/google/http.ex@@ -108,25 +108,27 @@ defmodule Plausible.Google.HTTP dodimensionFilterGroups: filter_groups})- "https://www.googleapis.com/webmasters/v3/sites/#{property}/searchAnalytics/query"- |> HTTPoison.post!(params,- "Content-Type": "application/json",- Authorization: "Bearer #{access_token}"+ :post+ |> Finch.build(+ "https://www.googleapis.com/webmasters/v3/sites/#{property}/searchAnalytics/query",+ [{"Authorization", "Bearer #{access_token}"}],+ params)+ |> Finch.request(Plausible.Finch)|> case do- %{status_code: 200, body: body} ->+ {:ok, %{status: 200, body: body}} ->{:ok, Jason.decode!(body)}- %{status_code: 401, body: body} ->+ {:ok, %{status: 401, body: body}} ->Sentry.capture_message("Error fetching Google queries", extra: Jason.decode!(body)){:error, :invalid_credentials}- %{status_code: 403, body: body} ->+ {:ok, %{status: 403, body: body}} ->body = Jason.decode!(body)Sentry.capture_message("Error fetching Google queries", extra: body){:error, get_in(body, ["error", "message"])}- %{body: body} ->+ {:ok, %{body: body}} ->Sentry.capture_message("Error fetching Google queries", extra: Jason.decode!(body)){:error, :unknown}enddiff --git a/test/plausible/google/api_test.exs b/test/plausible/google/api_test.exsindex 47f5c09c05..d4ee6ccbab 100644--- a/test/plausible/google/api_test.exs+++ b/test/plausible/google/api_test.exs@@ -1,18 +1,19 @@defmodule Plausible.Google.ApiTest douse Plausible.DataCase, async: true+ use ExVCR.Mock, adapter: ExVCR.Adapter.Finchalias Plausible.Google.Apiimport Plausible.TestUtilsimport Double- @ok_response File.read!("fixture/ga_batch_report.json")-- def start_buffer(_setup_args) do- {:ok, pid} = Plausible.Google.Buffer.start_link()- {:ok, buffer: pid}- end+ setup [:create_user, :create_new_site]describe "fetch_and_persist/4" do- setup [:create_user, :create_new_site, :start_buffer]+ @ok_response File.read!("fixture/ga_batch_report.json")++ setup do+ {:ok, pid} = Plausible.Google.Buffer.start_link()+ {:ok, buffer: pid}+ endtest "will fetch and persist import data from Google Analytics", %{site: site, buffer: buffer} dofinch_double =@@ -84,4 +85,66 @@ defmodule Plausible.Google.ApiTest doassert_receive({Finch, :request, [_, _]})endend++ describe "fetch_stats/3" do+ test "returns name and visitor count", %{user: user, site: site} do+ use_cassette "google_analytics_stats", match_requests_on: [:request_body] do+ insert(:google_auth,+ user: user,+ site: site,+ property: "sc-domain:dummy.test",+ expires: NaiveDateTime.add(NaiveDateTime.utc_now(), 3600)+ )++ query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}++ assert {:ok,+ [+ %{name: ["keyword1", "keyword2"], visitors: 25},+ %{name: ["keyword3", "keyword4"], visitors: 15}+ ]} = Plausible.Google.Api.fetch_stats(site, query, 5)+ end+ end++ test "returns next page when page argument is set", %{user: user, site: site} do+ use_cassette "google_analytics_stats#with_page", match_requests_on: [:request_body] do+ insert(:google_auth,+ user: user,+ site: site,+ property: "sc-domain:dummy.test",+ expires: NaiveDateTime.add(NaiveDateTime.utc_now(), 3600)+ )++ query = %Plausible.Stats.Query{+ filters: %{"page" => 5},+ date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])+ }++ assert {:ok,+ [+ %{name: ["keyword1", "keyword2"], visitors: 25},+ %{name: ["keyword3", "keyword4"], visitors: 15}+ ]} = Plausible.Google.Api.fetch_stats(site, query, 5)+ end+ end++ test "defaults first page when page argument is not set", %{user: user, site: site} do+ use_cassette "google_analytics_stats#without_page", match_requests_on: [:request_body] do+ insert(:google_auth,+ user: user,+ site: site,+ property: "sc-domain:dummy.test",+ expires: NaiveDateTime.add(NaiveDateTime.utc_now(), 3600)+ )++ query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}++ assert {:ok,+ [+ %{name: ["keyword1", "keyword2"], visitors: 25},+ %{name: ["keyword3", "keyword4"], visitors: 15}+ ]} = Plausible.Google.Api.fetch_stats(site, query, 5)+ end+ end+ endend