Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Prevent robots from looking too much into public dashboards (#3118)

hq1fea9bb32ee3b
diff --git a/lib/plausible_web/controllers/api/internal_controller.ex b/lib/plausible_web/controllers/api/internal_controller.ex
index b721bc9d08..c1e2707513 100644
--- a/lib/plausible_web/controllers/api/internal_controller.ex
+++ b/lib/plausible_web/controllers/api/internal_controller.ex
@@ -6,12 +6,14 @@ defmodule PlausibleWeb.Api.InternalController do
alias Plausible.Auth.User
def domain_status(conn, %{"domain" => domain}) do
- site = Sites.get_by_domain(domain)
-
- if Stats.has_pageviews?(site) do
+ with %User{id: user_id} <- conn.assigns[:current_user],
+ %Site{} = site <- Sites.get_by_domain(domain),
+ true <- Sites.has_admin_access?(user_id, site) || Auth.is_super_admin?(user_id),
+ true <- Stats.has_pageviews?(site) do
json(conn, "READY")
else
- json(conn, "WAITING")
+ _ ->
+ json(conn, "WAITING")
end
end
diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex
index 72b6ba9a4b..991a62c463 100644
--- a/lib/plausible_web/controllers/stats_controller.ex
+++ b/lib/plausible_web/controllers/stats_controller.ex
@@ -67,7 +67,7 @@ defmodule PlausibleWeb.StatsController do
funnels: Plausible.Funnels.list(site),
stats_start_date: stats_start_date,
native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at),
- title: "Plausible · " <> site.domain,
+ title: title(conn, site),
offer_email_report: offer_email_report,
demo: demo,
flags: get_flags(conn.assigns[:current_user]),
@@ -295,7 +295,7 @@ defmodule PlausibleWeb.StatsController do
funnels: Plausible.Funnels.list(shared_link.site),
stats_start_date: shared_link.site.stats_start_date,
native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at),
- title: "Plausible · " <> shared_link.site.domain,
+ title: title(conn, shared_link.site),
offer_email_report: false,
demo: false,
skip_plausible_tracking: true,
@@ -343,4 +343,12 @@ defmodule PlausibleWeb.StatsController do
!!is_or_nil
end
+
+ defp title(%{path_info: ["plausible.io"]}, _) do
+ "Plausible Analytics: Live Demo"
+ end
+
+ defp title(_conn, site) do
+ "Plausible · " <> site.domain
+ end
end
diff --git a/lib/plausible_web/plugs/no_robots.ex b/lib/plausible_web/plugs/no_robots.ex
new file mode 100644
index 0000000000..86f178fbd8
--- /dev/null
+++ b/lib/plausible_web/plugs/no_robots.ex
@@ -0,0 +1,29 @@
+defmodule PlausibleWeb.Plugs.NoRobots do
+ @moduledoc """
+ Rejects bot requests by any means available.
+
+ We're adding `x-robots-tag` to the response header and annotate the conn
+ with "noindex, nofollow" under `private.robots` key.
+
+ The only exception is, if the request is trying to access our live demo
+ at plausible.io/plausible.io - in which case we'll allow indexing, but deny
+ following links and skip the bot detection, in kind robots we trust.
+ """
+ @behaviour Plug
+ import Plug.Conn
+
+ @impl true
+ def init(opts), do: opts
+
+ @impl true
+ def call(conn, _opts \\ nil) do
+ conn =
+ if conn.path_info == ["plausible.io"] do
+ put_private(conn, :robots, "index, nofollow")
+ else
+ put_private(conn, :robots, "noindex, nofollow")
+ end
+
+ put_resp_header(conn, "x-robots-tag", conn.private.robots)
+ end
+end
diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex
index 018ab227fa..8c26db6f9a 100644
--- a/lib/plausible_web/router.ex
+++ b/lib/plausible_web/router.ex
@@ -8,6 +8,7 @@ defmodule PlausibleWeb.Router do
plug :fetch_session
plug :fetch_live_flash
plug :put_secure_browser_headers
+ plug PlausibleWeb.Plugs.NoRobots
plug PlausibleWeb.FirstLaunchPlug, redirect_to: "/register"
plug PlausibleWeb.SessionTimeoutPlug, timeout_after_seconds: @two_weeks_in_seconds
plug PlausibleWeb.AuthPlug
@@ -17,6 +18,7 @@ defmodule PlausibleWeb.Router do
pipeline :shared_link do
plug :accepts, ["html"]
plug :put_secure_browser_headers
+ plug PlausibleWeb.Plugs.NoRobots
end
pipeline :csrf do
@@ -33,6 +35,7 @@ defmodule PlausibleWeb.Router do
plug :accepts, ["json"]
plug :fetch_session
plug PlausibleWeb.AuthorizeSiteAccess
+ plug PlausibleWeb.Plugs.NoRobots
end
pipeline :public_api do
@@ -42,6 +45,7 @@ defmodule PlausibleWeb.Router do
pipeline :flags do
plug :accepts, ["html"]
plug :put_secure_browser_headers
+ plug PlausibleWeb.Plugs.NoRobots
plug :fetch_session
plug PlausibleWeb.CRMAuthPlug
end
@@ -50,7 +54,9 @@ defmodule PlausibleWeb.Router do
forward "/sent-emails", Bamboo.SentEmailViewerPlug
end
- use Kaffy.Routes, scope: "/crm", pipe_through: [PlausibleWeb.CRMAuthPlug]
+ use Kaffy.Routes,
+ scope: "/crm",
+ pipe_through: [PlausibleWeb.Plugs.NoRobots, PlausibleWeb.CRMAuthPlug]
scope path: "/flags" do
pipe_through :flags
@@ -59,7 +65,7 @@ defmodule PlausibleWeb.Router do
scope "/api/stats", PlausibleWeb.Api do
pipe_through :internal_stats_api
- get "/:domain//funnels/:id", StatsController, :funnel
+ get "/:domain/funnels/:id", StatsController, :funnel
get "/:domain/current-visitors", StatsController, :current_visitors
get "/:domain/main-graph", StatsController, :main_graph
get "/:domain/top-stats", StatsController, :top_stats
diff --git a/lib/plausible_web/templates/layout/app.html.eex b/lib/plausible_web/templates/layout/app.html.eex
index a9d79c8ee0..b58a7a2951 100644
--- a/lib/plausible_web/templates/layout/app.html.eex
+++ b/lib/plausible_web/templates/layout/app.html.eex
@@ -8,6 +8,7 @@
<%= if Plausible.Funnels.enabled_for?(@conn.assigns[:current_user]) do %>
<meta name="csrf-token" content="<%= Plug.CSRFProtection.get_csrf_token() %>" />
<meta name="websocket-url" content="<%= websocket_url() %>" />
+ <meta name="robots" content="<%= @conn.private.robots %>" />
<% end %>
<link rel="icon" type="image/png" sizes="32x32" href="<%= PlausibleWeb.Router.Helpers.static_path(@conn, "/images/icon/plausible_favicon.png") %>">
<link rel="apple-touch-icon" href="/images/icon/apple-touch-icon.png">
diff --git a/lib/plausible_web/templates/layout/focus.html.eex b/lib/plausible_web/templates/layout/focus.html.eex
index 3cfa049fa6..64f20ad5e7 100644
--- a/lib/plausible_web/templates/layout/focus.html.eex
+++ b/lib/plausible_web/templates/layout/focus.html.eex
@@ -5,6 +5,7 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge"/>
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
<meta name="description" content="A lightweight, non-intrusive alternative to Google Analytics."/>
+ <meta name="robots" content="<%= @conn.private.robots %>" />
<link rel="icon" type="image/png" sizes="32x32" href="<%= PlausibleWeb.Router.Helpers.static_path(@conn, "/images/icon/plausible_favicon.png") %>">
<title><%= assigns[:title] || "Plausible · Web analytics" %></title>
<link rel="stylesheet" href="<%= Routes.static_path(@conn, "/css/app.css") %>"/>
diff --git a/test/plausible_web/controllers/api/internal_controller_test.exs b/test/plausible_web/controllers/api/internal_controller_test.exs
index c7ce0800ec..2a2e5cd489 100644
--- a/test/plausible_web/controllers/api/internal_controller_test.exs
+++ b/test/plausible_web/controllers/api/internal_controller_test.exs
@@ -20,6 +20,21 @@ defmodule PlausibleWeb.Api.InternalControllerTest do
assert json_response(conn, 200) == "READY"
end
+
+ test "is WAITING when unauthenticated", %{user: user} do
+ site = insert(:site, members: [user])
+ Plausible.TestUtils.create_pageviews([%{site: site}])
+
+ conn = get(build_conn(), "/api/#{site.domain}/status")
+
+ assert json_response(conn, 200) == "WAITING"
+ end
+
+ test "is WAITING when non-existing site", %{conn: conn} do
+ conn = get(conn, "/api/example.com/status")
+
+ assert json_response(conn, 200) == "WAITING"
+ end
end
describe "GET /api/sites" do
diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs
index 7529d6367e..048e2d6313 100644
--- a/test/plausible_web/controllers/stats_controller_test.exs
+++ b/test/plausible_web/controllers/stats_controller_test.exs
@@ -1,6 +1,7 @@
defmodule PlausibleWeb.StatsControllerTest do
use PlausibleWeb.ConnCase, async: true
use Plausible.Repo
+ import Plausible.Test.Support.HTML
describe "GET /:website - anonymous user" do
test "public site - shows site stats", %{conn: conn} do
@@ -8,7 +9,31 @@ defmodule PlausibleWeb.StatsControllerTest do
populate_stats(site, [build(:pageview)])
conn = get(conn, "/#{site.domain}")
- assert html_response(conn, 200) =~ "stats-react-container"
+ resp = html_response(conn, 200)
+ assert element_exists?(resp, "div#stats-react-container")
+
+ assert ["noindex, nofollow"] ==
+ resp
+ |> find("meta[name=robots]")
+ |> Floki.attribute("content")
+
+ assert text_of_element(resp, "title") == "Plausible · #{site.domain}"
+ end
+
+ test "plausible.io live demo - shows site stats", %{conn: conn} do
+ site = insert(:site, domain: "plausible.io", public: true)
+ populate_stats(site, [build(:pageview)])
+
+ conn = get(conn, "/#{site.domain}")
+ resp = html_response(conn, 200)
+ assert element_exists?(resp, "div#stats-react-container")
+
+ assert ["index, nofollow"] ==
+ resp
+ |> find("meta[name=robots]")
+ |> Floki.attribute("content")
+
+ assert text_of_element(resp, "title") == "Plausible Analytics: Live Demo"
end
test "public site - shows waiting for first pageview", %{conn: conn} do
diff --git a/test/plausible_web/plugs/no_robots_test.exs b/test/plausible_web/plugs/no_robots_test.exs
new file mode 100644
index 0000000000..1897b4c870
--- /dev/null
+++ b/test/plausible_web/plugs/no_robots_test.exs
@@ -0,0 +1,36 @@
+defmodule PlausibleWeb.Plugs.NoRobotsTest do
+ use Plausible.DataCase, async: true
+ use Plug.Test
+
+ alias PlausibleWeb.Plugs.NoRobots
+
+ test "non-bots pass - when no user agent is supplied" do
+ conn = :get |> conn("/") |> NoRobots.call()
+ assert get_resp_header(conn, "x-robots-tag") == ["noindex, nofollow"]
+ assert get_resp_header(conn, "x-plausible-forbidden-reason") == []
+ assert conn.private.robots == "noindex, nofollow"
+
+ refute conn.halted
+ refute conn.status
+ end
+
+ test "non-bots pass - when user agent is supplied" do
+ conn = :get |> conn("/") |> NoRobots.call()
+ assert get_resp_header(conn, "x-robots-tag") == ["noindex, nofollow"]
+ assert get_resp_header(conn, "x-plausible-forbidden-reason") == []
+ assert conn.private.robots == "noindex, nofollow"
+
+ refute conn.halted
+ refute conn.status
+ end
+
+ test "writes index, nofollow for plausible.io live demo" do
+ conn = :get |> conn("/plausible.io") |> NoRobots.call()
+
+ assert get_resp_header(conn, "x-robots-tag") == ["index, nofollow"]
+ assert get_resp_header(conn, "x-plausible-forbidden-reason") == []
+
+ refute conn.halted
+ refute conn.status
+ end
+end