Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Filtering Search Console keywords (#4077)
Uku Taht06e8118dab1d
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6925d3355d..179be7be0a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -42,6 +42,7 @@ All notable changes to this project will be documented in this file.- Add Yesterday as an time range option in the dashboard- Add support for importing Google Analytics 4 data- Import custom events from Google Analytics 4+- Ability to filter Search Console keywords by page, country and device plausible/analytics#4077- Add `DATA_DIR` env var for exports/imports plausible/analytics#4100### Removeddiff --git a/assets/js/dashboard/stats/modals/exit-pages.js b/assets/js/dashboard/stats/modals/exit-pages.jsindex da2135d799..f8b2f71b0a 100644--- a/assets/js/dashboard/stats/modals/exit-pages.js+++ b/assets/js/dashboard/stats/modals/exit-pages.js@@ -4,7 +4,7 @@ import { withRouter } from 'react-router-dom'import Modal from './modal'import * as api from '../../api'-import numberFormatter from '../../util/number-formatter'+import numberFormatter, {percentageFormatter} from '../../util/number-formatter'import { parseQuery } from '../../query'import { trimURL } from '../../util/url'class ExitPagesModal extends React.Component {@@ -34,14 +34,6 @@ class ExitPagesModal extends React.Component {this.setState({ loading: true, page: this.state.page + 1 }, this.loadPages.bind(this))}- formatPercentage(number) {- if (typeof (number) === 'number') {- return number + '%'- } else {- return '-'- }- }-showConversionRate() {return !!this.state.query.filters.goal}@@ -74,7 +66,7 @@ class ExitPagesModal extends React.Component {{this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(page.total_visitors)}</td>}<td className="p-2 w-32 font-medium" align="right">{numberFormatter(page.visitors)}</td>{this.showExtra() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(page.visits)}</td>}- {this.showExtra() && <td className="p-2 w-32 font-medium" align="right">{this.formatPercentage(page.exit_rate)}</td>}+ {this.showExtra() && <td className="p-2 w-32 font-medium" align="right">{percentageFormatter(page.exit_rate)}</td>}{this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(page.conversion_rate)}%</td>}</tr>)diff --git a/assets/js/dashboard/stats/modals/google-keywords.js b/assets/js/dashboard/stats/modals/google-keywords.jsindex 3856ccd651..39ba7f3b8f 100644--- a/assets/js/dashboard/stats/modals/google-keywords.js+++ b/assets/js/dashboard/stats/modals/google-keywords.js@@ -3,7 +3,7 @@ import { Link, withRouter } from 'react-router-dom'import Modal from './modal'import * as api from '../../api'-import numberFormatter from '../../util/number-formatter'+import numberFormatter, { percentageFormatter } from '../../util/number-formatter'import {parseQuery} from '../../query'import RocketIcon from './rocket-icon'@@ -17,49 +17,31 @@ class GoogleKeywordsModal extends React.Component {}componentDidMount() {- if (this.state.query.filters.goal) {- api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/goal/referrers/Google`, this.state.query, {limit: 100})- .then((res) => this.setState({- loading: false,- searchTerms: res.search_terms,- totalVisitors: res.total_visitors,- notConfigured: res.not_configured,- isOwner: res.is_owner- }))- } else {- api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/referrers/Google`, this.state.query, {limit: 100})- .then((res) => this.setState({- loading: false,- searchTerms: res.search_terms,- totalVisitors: res.total_visitors,- notConfigured: res.not_configured,- isOwner: res.is_owner- }))- }+ api.get(`/api/stats/${encodeURIComponent(this.props.site.domain)}/referrers/Google`, this.state.query, {limit: 100})+ .then((res) => this.setState({+ loading: false,+ searchTerms: res.search_terms,+ notConfigured: res.not_configured,+ isOwner: res.is_owner+ }))}renderTerm(term) {return (<React.Fragment key={term.name}>-<tr className="text-sm dark:text-gray-200" key={term.name}>- <td className="p-2 truncate">{term.name}</td>+ <td className="p-2">{term.name}</td><td className="p-2 w-32 font-medium" align="right">{numberFormatter(term.visitors)}</td>+ <td className="p-2 w-32 font-medium" align="right">{numberFormatter(term.impressions)}</td>+ <td className="p-2 w-32 font-medium" align="right">{percentageFormatter(term.ctr)}</td>+ <td className="p-2 w-32 font-medium" align="right">{numberFormatter(term.position)}</td></tr></React.Fragment>)}renderKeywords() {- if (this.state.query.filters.goal) {- return (- <div className="text-center text-gray-700 dark:text-gray-300 mt-6">- <RocketIcon />- <div className="text-lg">Sorry, we cannot show which keywords converted best for goal <b>{this.state.query.filters.goal}</b></div>- <div className="text-lg">Google does not share this information</div>- </div>- )- } else if (this.state.notConfigured) {+ if (this.state.notConfigured) {if (this.state.isOwner) {return (<div className="text-center text-gray-700 dark:text-gray-300 mt-6">@@ -84,7 +66,10 @@ class GoogleKeywordsModal extends React.Component {<thead><tr><th className="p-2 w-48 md:w-56 lg:w-1/3 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="left">Search Term</th>- <th className="p-2 w-32 lg:w-1/2 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">Visitors</th>+ <th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">Visitors</th>+ <th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">Impressions</th>+ <th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">CTR</th>+ <th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">Position</th></tr></thead><tbody>@@ -102,14 +87,6 @@ class GoogleKeywordsModal extends React.Component {}}- renderGoalText() {- if (this.state.query.filters.goal) {- return (- <h1 className="text-xl font-semibold text-gray-500 dark:text-gray-200 leading-none">completed {this.state.query.filters.goal}</h1>- )- }- }-renderBody() {if (this.state.loading) {return (@@ -122,10 +99,6 @@ class GoogleKeywordsModal extends React.Component {<div className="my-4 border-b border-gray-300 dark:border-gray-500"></div><main className="modal__content">- <h1 className="text-xl font-semibold mb-0 leading-none dark:text-gray-200">- {this.state.totalVisitors} visitors from Google<br />- </h1>- {this.renderGoalText()}{ this.renderKeywords() }</main></React.Fragment>diff --git a/assets/js/dashboard/stats/sources/search-terms.js b/assets/js/dashboard/stats/sources/search-terms.jsindex 14863c67d2..2046e3db59 100644--- a/assets/js/dashboard/stats/sources/search-terms.js+++ b/assets/js/dashboard/stats/sources/search-terms.js@@ -39,7 +39,8 @@ export default class SearchTerms extends React.Component {loading: false,searchTerms: res.search_terms || [],notConfigured: res.not_configured,- isAdmin: res.is_admin+ isAdmin: res.is_admin,+ unsupportedFilters: res.unsupported_filters})).catch((error) =>{this.setState({ loading: false, searchTerms: [], notConfigured: true, error: true, isAdmin: error.payload.is_admin })@@ -68,21 +69,19 @@ export default class SearchTerms extends React.Component {}renderList() {- if (this.props.query.filters.goal) {+ if (this.state.unsupportedFilters) {return (<div className="text-center text-gray-700 dark:text-gray-300 text-sm mt-20"><RocketIcon />- <div>Sorry, we cannot show which keywords converted best for goal <b>{this.props.query.filters.goal}</b></div>- <div>Google does not share this information</div>+ <div>Unable to fetch keyword data from Search Console because it does not support the current set of filters</div></div>)-} else if (this.state.notConfigured) {return (<div className="text-center text-gray-700 dark:text-gray-300 text-sm mt-20"><RocketIcon /><div>- This site is not connected to Search Console so we cannot show the search phrases.+ This site is not connected to Search Console so we cannot show the search terms{this.state.isAdmin && this.state.error && <><br/><br/><p>Please click below to connect your Search Console account.</p></>}</div>{this.state.isAdmin && <a href={`/${encodeURIComponent(this.props.site.domain)}/settings/integrations`} className="button mt-4">Connect with Google</a> }@@ -103,9 +102,8 @@ export default class SearchTerms extends React.Component {)} else {return (- <div className="text-center text-gray-700 dark:text-gray-300 text-sm mt-20">- <RocketIcon />- <div>No search terms were found for this period. Please adjust or extend your time range. Check <a href="https://plausible.io/docs/google-search-console-integration#i-dont-see-google-search-query-data-in-my-dashboard" target="_blank" rel="noreferrer" className="hover:underline text-indigo-700 dark:text-indigo-500">our documentation</a> for more details.</div>+ <div className="text-center text-gray-700 dark:text-gray-300 ">+ <div className="mt-44 mx-auto font-medium text-gray-500 dark:text-gray-400">No data yet</div></div>)}diff --git a/assets/js/dashboard/util/number-formatter.js b/assets/js/dashboard/util/number-formatter.jsindex c724525ef8..48ced1e436 100644--- a/assets/js/dashboard/util/number-formatter.js+++ b/assets/js/dashboard/util/number-formatter.js@@ -49,3 +49,11 @@ export function durationFormatter(duration) {return `${seconds}s`}}++export function percentageFormatter(number) {+ if (typeof (number) === 'number') {+ return number + '%'+ } else {+ return '-'+ }+}diff --git a/fixture/http_mocks/google_analytics_stats#without_page.json b/fixture/http_mocks/google_analytics_stats#without_page.jsondeleted file mode 100644index 546cc140de..0000000000--- a/fixture/http_mocks/google_analytics_stats#without_page.json+++ /dev/null@@ -1,41 +0,0 @@-[- {- "status": 200,- "url": "https://www.googleapis.com/webmasters/v3/sites/sc-domain%3Adummy.test/searchAnalytics/query",- "method": "post",- "request_body": {- "dimensionFilterGroups": {},- "dimensions": [- "query"- ],- "endDate": "2022-01-05",- "rowLimit": 5,- "startDate": "2022-01-01"- },- "response_body": {- "responseAggregationType": "auto",- "rows": [- {- "clicks": 25.0,- "ctr": 0.3,- "impressions": 50.0,- "keys": [- "keyword1",- "keyword2"- ],- "position": 2.0- },- {- "clicks": 15.0,- "ctr": 0.5,- "impressions": 25.0,- "keys": [- "keyword3",- "keyword4"- ],- "position": 4.0- }- ]- }- }-]\ No newline at end of filediff --git a/fixture/http_mocks/google_analytics_stats.json b/fixture/http_mocks/google_analytics_stats.jsonindex 546cc140de..431b8fa2a7 100644--- a/fixture/http_mocks/google_analytics_stats.json+++ b/fixture/http_mocks/google_analytics_stats.json@@ -4,7 +4,7 @@"url": "https://www.googleapis.com/webmasters/v3/sites/sc-domain%3Adummy.test/searchAnalytics/query","method": "post","request_body": {- "dimensionFilterGroups": {},+ "dimensionFilterGroups": [],"dimensions": ["query"],@@ -16,26 +16,20 @@"responseAggregationType": "auto","rows": [{- "clicks": 25.0,- "ctr": 0.3,- "impressions": 50.0,- "keys": [- "keyword1",- "keyword2"- ],- "position": 2.0+ "clicks": 25,+ "ctr": 0.3679,+ "impressions": 50,+ "keys": ["keyword1"],+ "position": 2.2312312},{- "clicks": 15.0,+ "clicks": 15,"ctr": 0.5,- "impressions": 25.0,- "keys": [- "keyword3",- "keyword4"- ],+ "impressions": 25,+ "keys": ["keyword3"],"position": 4.0}]}}-]\ No newline at end of file+]diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.exindex 3e863c2f82..3fd63c87f7 100644--- a/lib/plausible/google/api.ex+++ b/lib/plausible/google/api.ex@@ -6,6 +6,7 @@ defmodule Plausible.Google.API douse Timexalias Plausible.Google.HTTP+ alias Plausible.Google.SearchConsolerequire Logger@@ -74,21 +75,26 @@ defmodule Plausible.Google.API doenddef fetch_stats(site, %{filters: %{} = filters, date_range: date_range}, limit) do- with site <- Plausible.Repo.preload(site, :google_auth),+ with {:ok, site} <- ensure_search_console_property(site),{:ok, access_token} <- maybe_refresh_token(site.google_auth),+ {:ok, search_console_filters} <-+ SearchConsole.Filters.transform(site.google_auth.property, filters),{:ok, stats} <-HTTP.list_stats(access_token,site.google_auth.property,date_range,limit,- filters["page"]+ search_console_filters) dostats|> Map.get("rows", [])- |> Enum.filter(fn row -> row["clicks"] > 0 end)- |> Enum.map(fn row -> %{name: row["keys"], visitors: round(row["clicks"])} end)+ |> Enum.map(&search_console_row/1)|> then(&{:ok, &1})+ else+ :google_property_not_configured -> {:error, :google_property_not_configured}+ :unsupported_filters -> {:error, :unsupported_filters}+ {:error, error} -> {:error, error}endend@@ -142,6 +148,44 @@ defmodule Plausible.Google.API doTimex.before?(expires_at, thirty_seconds_ago)end+ defp ensure_search_console_property(site) do+ site = Plausible.Repo.preload(site, :google_auth)++ if site.google_auth && site.google_auth.property do+ {:ok, site}+ else+ :google_property_not_configured+ end+ end++ defp search_console_row(row) do+ %{+ # We always request just one dimension at a time (`query`)+ name: row["keys"] |> List.first(),+ visitors: row["clicks"],+ impressions: row["impressions"],+ ctr: rounded_ctr(row["ctr"]),+ position: rounded_position(row["position"])+ }+ end++ defp rounded_ctr(ctr) do+ {:ok, decimal} = Decimal.cast(ctr)++ decimal+ |> Decimal.mult(100)+ |> Decimal.round(1)+ |> Decimal.to_float()+ end++ defp rounded_position(position) do+ {:ok, decimal} = Decimal.cast(position)++ decimal+ |> Decimal.round(1)+ |> Decimal.to_float()+ end+defp client_id() doKeyword.fetch!(Application.get_env(:plausible, :google), :client_id)enddiff --git a/lib/plausible/google/http.ex b/lib/plausible/google/http.exindex be55fbcf5e..c24808764b 100644--- a/lib/plausible/google/http.ex+++ b/lib/plausible/google/http.ex@@ -39,26 +39,18 @@ defmodule Plausible.Google.HTTP doresponse.bodyend- def list_stats(access_token, property, date_range, limit, page \\ nil) do- property = URI.encode_www_form(property)-- filter_groups =- if page do- url = property_base_url(property)- [%{filters: [%{dimension: "page", expression: "https://#{url}#{page}"}]}]- else- %{}- end-+ def list_stats(access_token, property, date_range, limit, search_console_filters) doparams = %{startDate: Date.to_iso8601(date_range.first),endDate: Date.to_iso8601(date_range.last),dimensions: ["query"],rowLimit: limit,- dimensionFilterGroups: filter_groups+ dimensionFilterGroups: search_console_filters}- url = "#{api_url()}/webmasters/v3/sites/#{property}/searchAnalytics/query"+ url =+ "#{api_url()}/webmasters/v3/sites/#{URI.encode_www_form(property)}/searchAnalytics/query"+headers = [{"Authorization", "Bearer #{access_token}"}]case HTTPClient.impl().post(url, headers, params) do@@ -78,9 +70,6 @@ defmodule Plausible.Google.HTTP doendend- defp property_base_url("sc-domain:" <> domain), do: "https://" <> domain- defp property_base_url(url), do: url-def refresh_auth_token(refresh_token) dourl = "#{api_url()}/oauth2/v4/token"headers = [{"content-type", "application/x-www-form-urlencoded"}]diff --git a/lib/plausible/google/search_console/filters.ex b/lib/plausible/google/search_console/filters.exnew file mode 100644index 0000000000..4e0e00d875--- /dev/null+++ b/lib/plausible/google/search_console/filters.ex@@ -0,0 +1,83 @@+defmodule Plausible.Google.SearchConsole.Filters do+ @moduledoc false+ import Plausible.Stats.Base, only: [page_regex: 1]++ def transform(property, plausible_filters) do+ plausible_filters = Map.drop(plausible_filters, ["visit:source"])++ search_console_filters =+ Enum.reduce_while(plausible_filters, [], fn plausible_filter, search_console_filters ->+ case transform_filter(property, plausible_filter) do+ :unsupported -> {:halt, :unsupported_filters}+ search_console_filter -> {:cont, [search_console_filter | search_console_filters]}+ end+ end)++ case search_console_filters do+ :unsupported_filters -> :unsupported_filters+ [] -> {:ok, []}+ filters when is_list(filters) -> {:ok, [%{filters: filters}]}+ end+ end++ defp transform_filter(property, {"event:page", filter}) do+ transform_filter(property, {"visit:entry_page", filter})+ end++ defp transform_filter(property, {"visit:entry_page", {:is, page}}) when is_binary(page) do+ %{dimension: "page", expression: property_url(property, page)}+ end++ defp transform_filter(property, {"visit:entry_page", {:member, pages}}) when is_list(pages) do+ expression =+ Enum.map_join(pages, "|", fn page -> property_url(property, Regex.escape(page)) end)++ %{dimension: "page", operator: "includingRegex", expression: expression}+ end++ defp transform_filter(property, {"visit:entry_page", {:matches, page}}) when is_binary(page) do+ page = page_regex(property_url(property, page))+ %{dimension: "page", operator: "includingRegex", expression: page}+ end++ defp transform_filter(property, {"visit:entry_page", {:matches_member, pages}})+ when is_list(pages) do+ expression =+ Enum.map_join(pages, "|", fn page -> page_regex(property_url(property, page)) end)++ %{dimension: "page", operator: "includingRegex", expression: expression}+ end++ defp transform_filter(_property, {"visit:screen", {:is, device}}) when is_binary(device) do+ %{dimension: "device", expression: search_console_device(device)}+ end++ defp transform_filter(_property, {"visit:screen", {:member, devices}}) when is_list(devices) do+ expression = devices |> Enum.join("|")+ %{dimension: "device", operator: "includingRegex", expression: expression}+ end++ defp transform_filter(_property, {"visit:country", {:is, country}}) when is_binary(country) do+ %{dimension: "country", expression: search_console_country(country)}+ end++ defp transform_filter(_property, {"visit:country", {:member, countries}})+ when is_list(countries) do+ expression = Enum.map_join(countries, "|", &search_console_country/1)+ %{dimension: "country", operator: "includingRegex", expression: expression}+ end++ defp transform_filter(_, _filter), do: :unsupported++ defp property_url("sc-domain:" <> domain, page), do: "https://" <> domain <> page+ defp property_url(url, page), do: url <> page++ defp search_console_device("Desktop"), do: "DESKTOP"+ defp search_console_device("Mobile"), do: "MOBILE"+ defp search_console_device("Tablet"), do: "TABLET"++ defp search_console_country(alpha_2) do+ country = Location.Country.get_country(alpha_2)+ country.alpha_3+ end+enddiff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 66659f3e5d..dc2a562cdd 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -680,36 +680,29 @@ defmodule PlausibleWeb.Api.StatsController doenddef referrer_drilldown(conn, %{"referrer" => "Google"} = params) do- site = conn.assigns[:site] |> Repo.preload(:google_auth)-- query =- Query.from(site, params)- |> Query.put_filter("visit:source", "Google")-- search_terms =- if site.google_auth && site.google_auth.property && !query.filters["goal"] do- google_api().fetch_stats(site, query, params["limit"] || 9)- end+ site = conn.assigns[:site]- %{:visitors => %{value: total_visitors}} = Stats.aggregate(site, query, [:visitors])+ query = Query.from(site, params)user_id = get_session(conn, :current_user_id)is_admin = user_id && Plausible.Sites.has_admin_access?(user_id, site)- case search_terms do- nil ->- json(conn, %{not_configured: true, is_admin: is_admin, total_visitors: total_visitors})+ case google_api().fetch_stats(site, query, params["limit"] || 9) do+ {:error, :google_propery_not_configured} ->+ json(conn, %{not_configured: true, is_admin: is_admin})++ {:error, :unsupported_filters} ->+ json(conn, %{unsupported_filters: true}){:ok, terms} ->- json(conn, %{search_terms: terms, total_visitors: total_visitors})+ json(conn, %{search_terms: terms}){:error, _} ->conn|> put_status(502)|> json(%{not_configured: true,- is_admin: is_admin,- total_visitors: total_visitors+ is_admin: is_admin})endenddiff --git a/test/plausible/google/api_test.exs b/test/plausible/google/api_test.exsindex 99bc350c6d..eca4fab00e 100644--- a/test/plausible/google/api_test.exs+++ b/test/plausible/google/api_test.exs@@ -245,7 +245,7 @@ defmodule Plausible.Google.APITest do"https://www.googleapis.com/webmasters/v3/sites/sc-domain%3Adummy.test/searchAnalytics/query",[{"Authorization", "Bearer 123"}],%{- dimensionFilterGroups: %{},+ dimensionFilterGroups: [],dimensions: ["query"],endDate: "2022-01-05",rowLimit: 5,@@ -301,29 +301,31 @@ defmodule Plausible.Google.APITest doendend- describe "fetch_stats/3 with VCR cassetes" do- test "returns name and visitor count", %{user: user, site: site} do- mock_http_with("google_analytics_stats.json")+ test "returns error when token refresh fails", %{user: user, site: site} do+ mock_http_with("google_analytics_auth#invalid_grant.json")- insert(:google_auth,- user: user,- site: site,- property: "sc-domain:dummy.test",- expires: NaiveDateTime.add(NaiveDateTime.utc_now(), 3600)- )+ insert(:google_auth,+ user: user,+ site: site,+ property: "sc-domain:dummy.test",+ access_token: "*****",+ refresh_token: "*****",+ expires: NaiveDateTime.add(NaiveDateTime.utc_now(), -3600)+ )- query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}+ query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}- assert {:ok,- [- %{name: ["keyword1", "keyword2"], visitors: 25},- %{name: ["keyword3", "keyword4"], visitors: 15}- ]} = Google.API.fetch_stats(site, query, 5)- end+ assert {:error, "invalid_grant"} = Google.API.fetch_stats(site, query, 5)+ end++ test "returns error when google auth not configured", %{site: site} do+ query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}- test "returns next page when page argument is set", %{user: user, site: site} do- mock_http_with("google_analytics_stats#with_page.json")+ assert {:error, :google_property_not_configured} = Google.API.fetch_stats(site, query, 5)+ end+ describe "fetch_stats/3 with valid auth" do+ setup %{user: user, site: site} doinsert(:google_auth,user: user,site: site,@@ -331,52 +333,64 @@ defmodule Plausible.Google.APITest doexpires: NaiveDateTime.add(NaiveDateTime.utc_now(), 3600))- query = %Plausible.Stats.Query{- filters: %{"page" => 5},- date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])- }-- assert {:ok,- [- %{name: ["keyword1", "keyword2"], visitors: 25},- %{name: ["keyword3", "keyword4"], visitors: 15}- ]} = Google.API.fetch_stats(site, query, 5)+ :okend- test "defaults first page when page argument is not set", %{user: user, site: site} do- mock_http_with("google_analytics_stats#without_page.json")-- insert(:google_auth,- user: user,- site: site,- property: "sc-domain:dummy.test",- expires: NaiveDateTime.add(NaiveDateTime.utc_now(), 3600)- )+ test "returns name and visitor count", %{site: site} do+ mock_http_with("google_analytics_stats.json")query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}assert {:ok,[- %{name: ["keyword1", "keyword2"], visitors: 25},- %{name: ["keyword3", "keyword4"], visitors: 15}+ %{name: "keyword1", visitors: 25, ctr: 36.8, impressions: 50, position: 2.2},+ %{name: "keyword3", visitors: 15}]} = Google.API.fetch_stats(site, query, 5)end- test "returns error when token refresh fails", %{user: user, site: site} do- mock_http_with("google_analytics_auth#invalid_grant.json")-- insert(:google_auth,- user: user,- site: site,- property: "sc-domain:dummy.test",- access_token: "*****",- refresh_token: "*****",- expires: NaiveDateTime.add(NaiveDateTime.utc_now(), -3600)+ test "transforms page filters to search console format", %{site: site} do+ expect(+ Plausible.HTTPClient.Mock,+ :post,+ fn+ "https://www.googleapis.com/webmasters/v3/sites/sc-domain%3Adummy.test/searchAnalytics/query",+ [{"Authorization", "Bearer 123"}],+ %{+ dimensionFilterGroups: [+ %{filters: [%{expression: "https://dummy.test/page", dimension: "page"}]}+ ],+ dimensions: ["query"],+ endDate: "2022-01-05",+ rowLimit: 5,+ startDate: "2022-01-01"+ } ->+ {:ok, %Finch.Response{status: 200, body: %{"rows" => []}}}+ end)- query = %Plausible.Stats.Query{date_range: Date.range(~D[2022-01-01], ~D[2022-01-05])}+ query =+ Plausible.Stats.Query.from(site, %{+ "period" => "custom",+ "from" => "2022-01-01",+ "to" => "2022-01-05",+ "filters" => "event:page==/page"+ })++ assert {:ok, []} = Google.API.fetch_stats(site, query, 5)+ end- assert {:error, "invalid_grant"} = Google.API.fetch_stats(site, query, 5)+ test "returns :invalid filters when using filters that cannot be used in Search Console", %{+ site: site+ } do+ query =+ Plausible.Stats.Query.from(site, %{+ "period" => "custom",+ "from" => "2022-01-01",+ "to" => "2022-01-05",+ "filters" => "event:goal==Signup"+ })++ assert {:error, :unsupported_filters} = Google.API.fetch_stats(site, query, 5)endenddiff --git a/test/plausible/google/search_console/filters_test.exs b/test/plausible/google/search_console/filters_test.exsnew file mode 100644index 0000000000..7d73dacda8--- /dev/null+++ b/test/plausible/google/search_console/filters_test.exs@@ -0,0 +1,183 @@+defmodule Plausible.Google.SearchConsole.FiltersTest do+ alias Plausible.Google.SearchConsole.Filters+ use Plausible.DataCase, async: true++ test "transforms simple page filter" do+ filters = %{+ "visit:entry_page" => {:is, "/page"}+ }++ {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)++ assert transformed == [+ %{filters: [%{dimension: "page", expression: "https://plausible.io/page"}]}+ ]+ end++ test "transforms matches page filter" do+ filters = %{+ "visit:entry_page" => {:matches, "*page*"}+ }++ {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)++ assert transformed == [+ %{+ filters: [+ %{+ dimension: "page",+ operator: "includingRegex",+ expression: "^https://plausible\\.io.*page.*$"+ }+ ]+ }+ ]+ end++ test "transforms member page filter" do+ filters = %{+ "visit:entry_page" => {:member, ["/pageA", "/pageB"]}+ }++ {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)++ assert transformed == [+ %{+ filters: [+ %{+ dimension: "page",+ operator: "includingRegex",+ expression: "https://plausible.io/pageA|https://plausible.io/pageB"+ }+ ]+ }+ ]+ end++ test "transforms matches_member page filter" do+ filters = %{+ "visit:entry_page" => {:matches_member, ["/pageA*", "/pageB*"]}+ }++ {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)++ assert transformed == [+ %{+ filters: [+ %{+ dimension: "page",+ operator: "includingRegex",+ expression: "^https://plausible\\.io/pageA.*$|^https://plausible\\.io/pageB.*$"+ }+ ]+ }+ ]+ end++ test "transforms event:page exactly like visit:entry_page" do+ filters = %{+ "event:page" => {:matches_member, ["/pageA*", "/pageB*"]}+ }++ {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)++ assert transformed == [+ %{+ filters: [+ %{+ dimension: "page",+ operator: "includingRegex",+ expression: "^https://plausible\\.io/pageA.*$|^https://plausible\\.io/pageB.*$"+ }+ ]+ }+ ]+ end++ test "transforms simple visit:screen filter" do+ filters = %{+ "visit:screen" => {:is, "Desktop"}+ }++ {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)++ assert transformed == [%{filters: [%{dimension: "device", expression: "DESKTOP"}]}]+ end++ test "transforms member visit:screen filter" do+ filters = %{+ "visit:screen" => {:member, ["Mobile", "Tablet"]}+ }++ {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)++ assert transformed == [+ %{+ filters: [+ %{dimension: "device", operator: "includingRegex", expression: "Mobile|Tablet"}+ ]+ }+ ]+ end++ test "transforms simple visit:country filter to alpha3" do+ filters = %{+ "visit:country" => {:is, "EE"}+ }++ {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)++ assert transformed == [%{filters: [%{dimension: "country", expression: "EST"}]}]+ end++ test "transforms member visit:country filter" do+ filters = %{+ "visit:country" => {:member, ["EE", "PL"]}+ }++ {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)++ assert transformed == [+ %{+ filters: [+ %{dimension: "country", operator: "includingRegex", expression: "EST|POL"}+ ]+ }+ ]+ end++ test "filters can be combined" do+ filters = %{+ "visit:entry_page" => {:matches, "*web-analytics*"},+ "visit:screen" => {:is, "Desktop"},+ "visit:country" => {:member, ["EE", "PL"]}+ }++ {:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)++ assert transformed == [+ %{+ filters: [+ %{dimension: "device", expression: "DESKTOP"},+ %{+ dimension: "page",+ operator: "includingRegex",+ expression: "^https://plausible\\.io.*web\\-analytics.*$"+ },+ %{dimension: "country", operator: "includingRegex", expression: "EST|POL"}+ ]+ }+ ]+ end++ test "when unsupported filter is included the whole set becomes invalid" do+ filters = %{+ "visit:entry_page" => {:matches, "*web-analytics*"},+ "visit:screen" => {:is, "Desktop"},+ "visit:country" => {:member, ["EE", "PL"]},+ "visit:utm_medium" => {:is, "facebook"}+ }++ assert :unsupported_filters = Filters.transform("sc-domain:plausible.io", filters)+ end+enddiff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exsindex 4d4481c4c0..dfa9480c02 100644--- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs@@ -1606,9 +1606,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do]end- test "gets keywords from Google", %{conn: conn, user: user, site: site} do- insert(:google_auth, user: user, user: user, site: site, property: "sc-domain:example.com")-+ test "gets keywords from Google", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview,referrer_source: "DuckDuckGo",@@ -1627,10 +1625,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest doconn = get(conn, "/api/stats/#{site.domain}/referrers/Google?period=day"){:ok, terms} = Plausible.Google.API.Mock.fetch_stats(nil, nil, nil)- assert json_response(conn, 200) == %{- "total_visitors" => 2,- "search_terms" => terms- }+ assert json_response(conn, 200) == %{"search_terms" => terms}endtest "works when filter expression is provided for source", %{diff --git a/test/test_helper.exs b/test/test_helper.exsindex 924b75a3b7..9b1b07f7a8 100644--- a/test/test_helper.exs+++ b/test/test_helper.exs@@ -23,7 +23,7 @@ if :minio in Keyword.fetch!(ExUnit.configuration(), :include) doendif Mix.env() == :ce_test do- IO.puts("Test mode: Communnity Edition")+ IO.puts("Test mode: Community Edition")ExUnit.configure(exclude: [:slow, :minio, :ee_only])elseIO.puts("Test mode: Enterprise Edition")