Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

ScriptV2: TrackerScriptCache on ee (#5648)

Karl-Aksel Puulmanncf423dbf9951
diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex
index a0ab741377..ffc144a01a 100644
--- a/lib/plausible/application.ex
+++ b/lib/plausible/application.ex
@@ -128,22 +128,20 @@ defmodule Plausible.Application do
]
)
end,
- on_ce do
- warmed_cache(PlausibleWeb.TrackerScriptCache,
- adapter_opts: [
- n_lock_partitions: 1,
- ttl_check_interval: false,
- ets_options: [:bag, read_concurrency: true]
- ],
- warmers: [
- refresh_all:
- {PlausibleWeb.TrackerScriptCache.All,
- interval: :timer.minutes(180) + Enum.random(1..:timer.seconds(10))},
- refresh_updated_recently:
- {PlausibleWeb.TrackerScriptCache.RecentlyUpdated, interval: :timer.seconds(120)}
- ]
- )
- end,
+ warmed_cache(PlausibleWeb.TrackerScriptCache,
+ adapter_opts: [
+ n_lock_partitions: 1,
+ ttl_check_interval: false,
+ ets_options: [:bag, read_concurrency: true]
+ ],
+ warmers: [
+ refresh_all:
+ {PlausibleWeb.TrackerScriptCache.All,
+ interval: :timer.minutes(180) + Enum.random(1..:timer.seconds(10))},
+ refresh_updated_recently:
+ {PlausibleWeb.TrackerScriptCache.RecentlyUpdated, interval: :timer.seconds(30)}
+ ]
+ ),
Plausible.Ingestion.Counters,
Plausible.Session.Salts,
Supervisor.child_spec(Plausible.Event.WriteBuffer, id: Plausible.Event.WriteBuffer),
diff --git a/lib/plausible/telemetry/plausible_metrics.ex b/lib/plausible/telemetry/plausible_metrics.ex
index 1ba4a7d9b6..dd8d4ab25e 100644
--- a/lib/plausible/telemetry/plausible_metrics.ex
+++ b/lib/plausible/telemetry/plausible_metrics.ex
@@ -47,6 +47,24 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do
unit: {:native, :millisecond},
measurement: :duration
),
+ distribution(
+ metric_prefix ++ [:cache_warmer, :tracker_script, :refresh, :all],
+ event_name: PlausibleWeb.TrackerScriptCache.telemetry_event_refresh(:all),
+ reporter_options: [
+ buckets: [500, 1000, 2000, 5000, 10_000]
+ ],
+ unit: {:native, :millisecond},
+ measurement: :duration
+ ),
+ distribution(
+ metric_prefix ++ [:cache_warmer, :tracker_script, :refresh, :updated_recently],
+ event_name: PlausibleWeb.TrackerScriptCache.telemetry_event_refresh(:updated_recently),
+ reporter_options: [
+ buckets: [500, 1000, 2000, 5000, 10_000]
+ ],
+ unit: {:native, :millisecond},
+ measurement: :duration
+ ),
distribution(
metric_prefix ++ [:ingest, :events, :pipeline, :steps],
event_name: Ingestion.Event.telemetry_pipeline_step_duration(),
diff --git a/lib/plausible_web/controllers/api/system_controller.ex b/lib/plausible_web/controllers/api/system_controller.ex
index 95482e0d52..3fdd993fa9 100644
--- a/lib/plausible_web/controllers/api/system_controller.ex
+++ b/lib/plausible_web/controllers/api/system_controller.ex
@@ -22,6 +22,11 @@ defmodule PlausibleWeb.Api.SystemController do
end
@task_timeout 15_000
+ @critical_caches [
+ Plausible.Site.Cache,
+ Plausible.Shield.IPRuleCache,
+ PlausibleWeb.TrackerScriptCache
+ ]
def readiness(conn, _params) do
postgres_health_task =
Task.async(fn ->
@@ -54,8 +59,7 @@ defmodule PlausibleWeb.Api.SystemController do
end
cache_health =
- if postgres_health == "ok" and Plausible.Site.Cache.ready?() and
- Plausible.Shield.IPRuleCache.ready?() do
+ if postgres_health == "ok" and Enum.all?(@critical_caches, & &1.ready?()) do
"ok"
end
diff --git a/lib/plausible_web/plugs/tracker_plug.ex b/lib/plausible_web/plugs/tracker_plug.ex
index 9e464fdccb..2f067ffcd0 100644
--- a/lib/plausible_web/plugs/tracker_plug.ex
+++ b/lib/plausible_web/plugs/tracker_plug.ex
@@ -62,8 +62,8 @@ defmodule PlausibleWeb.TrackerPlug do
def telemetry_event(name), do: [:plausible, :tracker_script, :request, name]
defp request_tracker_script(id, conn) do
- case get_plausible_web_script(id) do
- {:ok, script_tag} ->
+ case PlausibleWeb.Tracker.get_plausible_main_script(id) do
+ script_tag when is_binary(script_tag) ->
:telemetry.execute(
telemetry_event(:v2),
%{},
@@ -82,7 +82,7 @@ defmodule PlausibleWeb.TrackerPlug do
|> send_resp(200, script_tag)
|> halt()
- {:error, :not_found} ->
+ nil ->
:telemetry.execute(
telemetry_event(:v2),
%{},
@@ -95,23 +95,6 @@ defmodule PlausibleWeb.TrackerPlug do
end
end
- defp get_plausible_web_script(id) do
- script =
- on_ee do
- # On cloud, we generate the script always on the fly relying on CDN caching
- PlausibleWeb.TrackerScriptCache.get_from_source(id)
- else
- # On self-hosted, we have a pre-warmed cache for the script
- PlausibleWeb.TrackerScriptCache.get(id)
- end
-
- if script do
- {:ok, script}
- else
- {:error, :not_found}
- end
- end
-
defp legacy_request_file(filename, files_available, conn) do
if filename && MapSet.member?(files_available, filename) do
location = Application.app_dir(:plausible, "priv/tracker/js/" <> filename)
diff --git a/lib/plausible_web/tracker.ex b/lib/plausible_web/tracker.ex
index 6a79af3dd9..8a98b42d3a 100644
--- a/lib/plausible_web/tracker.ex
+++ b/lib/plausible_web/tracker.ex
@@ -14,7 +14,36 @@ defmodule PlausibleWeb.Tracker do
@plausible_main_script File.read!(path)
@external_resource "priv/tracker/js/plausible-web.js"
- def plausible_main_script_tag(tracker_script_configuration) do
+ @spec get_plausible_main_script(String.t(), Keyword.t()) :: String.t() | nil
+ def get_plausible_main_script(id, cache_opts \\ []) do
+ on_ee do
+ # On cloud:
+ # 1. Check if tracker script ID is in the cache
+ # 2. If it is, generate the script on the fly
+ #
+ # Note that EE is relying on CDN caching the script
+ if PlausibleWeb.TrackerScriptCache.get(id, cache_opts) do
+ get_tracker_script_configuration(id)
+ |> build_script()
+ end
+ else
+ # On self-hosted, we have a pre-warmed cache for the script
+ PlausibleWeb.TrackerScriptCache.get(id, cache_opts)
+ end
+ end
+
+ # Exposed for testing
+ def plausible_main_config(tracker_script_configuration) do
+ %{
+ domain: tracker_script_configuration.site.domain,
+ endpoint: tracker_ingestion_endpoint(),
+ outboundLinks: tracker_script_configuration.outbound_links,
+ fileDownloads: tracker_script_configuration.file_downloads,
+ formSubmissions: tracker_script_configuration.form_submissions
+ }
+ end
+
+ def build_script(%TrackerScriptConfiguration{} = tracker_script_configuration) do
config_js_content =
tracker_script_configuration
|> plausible_main_config()
@@ -32,15 +61,7 @@ defmodule PlausibleWeb.Tracker do
|> String.replace("\"<%= @config_js %>\"", "{#{config_js_content}}")
end
- def plausible_main_config(tracker_script_configuration) do
- %{
- domain: tracker_script_configuration.site.domain,
- endpoint: tracker_ingestion_endpoint(),
- outboundLinks: tracker_script_configuration.outbound_links,
- fileDownloads: tracker_script_configuration.file_downloads,
- formSubmissions: tracker_script_configuration.form_submissions
- }
- end
+ def build_script(nil), do: nil
def update_script_configuration(site, config_update, changeset_type) do
Repo.transact(fn ->
@@ -108,6 +129,19 @@ defmodule PlausibleWeb.Tracker do
config
end
+ on_ee do
+ import Ecto.Query
+
+ defp get_tracker_script_configuration(id) do
+ from(t in TrackerScriptConfiguration,
+ where: t.id == ^id,
+ join: s in assoc(t, :site),
+ preload: [site: s]
+ )
+ |> Plausible.Repo.one()
+ end
+ end
+
# Sync plausible goals with the updated script config
defp sync_goals(site, original_config, updated_config) do
[:track_404_pages, :outbound_links, :file_downloads, :form_submissions]
diff --git a/lib/plausible_web/tracker_script_cache.ex b/lib/plausible_web/tracker_script_cache.ex
index 74ec817ba4..031aebbc57 100644
--- a/lib/plausible_web/tracker_script_cache.ex
+++ b/lib/plausible_web/tracker_script_cache.ex
@@ -1,10 +1,14 @@
defmodule PlausibleWeb.TrackerScriptCache do
@moduledoc """
- Cache for tracker script(s) for self-hosted Plausible instances.
+ Cache for tracker script.
+
+ On self-hosted instances, we cache the entire tracker script.
+ On EE instances, we cache valid tracker script ids to avoid database lookups.
"""
alias Plausible.Site.TrackerScriptConfiguration
import Ecto.Query
+ use Plausible
use Plausible.Cache
@cache_name :tracker_script_cache
@@ -37,7 +41,7 @@ defmodule PlausibleWeb.TrackerScriptCache do
case Plausible.Repo.one(query) do
%TrackerScriptConfiguration{} = tracker_script_configuration ->
- PlausibleWeb.Tracker.plausible_main_script_tag(tracker_script_configuration)
+ cache_content(tracker_script_configuration)
_ ->
nil
@@ -49,10 +53,17 @@ defmodule PlausibleWeb.TrackerScriptCache do
Enum.reduce(items, [], fn
tracker_script_configuration, acc ->
[
- {tracker_script_configuration.id,
- PlausibleWeb.Tracker.plausible_main_script_tag(tracker_script_configuration)}
+ {tracker_script_configuration.id, cache_content(tracker_script_configuration)}
| acc
]
end)
end
+
+ defp cache_content(tracker_script_configuration) do
+ if ee?() do
+ true
+ else
+ PlausibleWeb.Tracker.build_script(tracker_script_configuration)
+ end
+ end
end
diff --git a/test/plausible_web/tracker_script_cache_test.exs b/test/plausible_web/tracker_script_cache_test.exs
index 7a8a9b4e7d..31f0c8714b 100644
--- a/test/plausible_web/tracker_script_cache_test.exs
+++ b/test/plausible_web/tracker_script_cache_test.exs
@@ -1,4 +1,5 @@
defmodule PlausibleWeb.TrackerScriptCacheTest do
+ use Plausible
use Plausible.DataCase, async: true
use Plausible.Teams.Test
@@ -6,7 +7,7 @@ defmodule PlausibleWeb.TrackerScriptCacheTest do
alias PlausibleWeb.TrackerScriptCache
describe "public cache interface" do
- test "cache caches tracker script configurations", %{test: test} do
+ test "cache caches tracker scripts by id", %{test: test} do
{:ok, _} =
Supervisor.start_link(
[{TrackerScriptCache, [cache_name: test, child_id: :test_cache_tracker_script]}],
@@ -23,8 +24,13 @@ defmodule PlausibleWeb.TrackerScriptCacheTest do
assert TrackerScriptCache.size(test) == 1
- assert script_tag = TrackerScriptCache.get(config.id, force?: true, cache_name: test)
- assert is_binary(script_tag)
+ assert result = TrackerScriptCache.get(config.id, force?: true, cache_name: test)
+
+ on_ee do
+ assert result == true
+ else
+ assert is_binary(result)
+ end
refute TrackerScriptCache.get("nonexistent", cache_name: test, force?: true)
end
diff --git a/test/plausible_web/tracker_test.exs b/test/plausible_web/tracker_test.exs
index d727d610f3..78b6bb0530 100644
--- a/test/plausible_web/tracker_test.exs
+++ b/test/plausible_web/tracker_test.exs
@@ -1,38 +1,39 @@
defmodule PlausibleWeb.TrackerTest do
- use Plausible.DataCase, async: true
+ use Plausible.DataCase
use Oban.Testing, repo: Plausible.Repo
use Plausible
use Plausible.Teams.Test
alias Plausible.Site.TrackerScriptConfiguration
alias PlausibleWeb.Tracker
+ alias PlausibleWeb.TrackerScriptCache
+
+ @example_config %{
+ installation_type: :manual,
+ track_404_pages: true,
+ hash_based_routing: true,
+ file_downloads: false,
+ outbound_links: false,
+ pageview_props: false,
+ tagged_events: true,
+ revenue_tracking: false,
+ form_submissions: true
+ }
+
+ test "plausible_main_config/1" do
+ site = new_site()
+ tracker_script_configuration = create_config(site)
+
+ assert Tracker.plausible_main_config(tracker_script_configuration) == %{
+ domain: site.domain,
+ endpoint: "#{PlausibleWeb.Endpoint.url()}/api/event",
+ outboundLinks: false,
+ fileDownloads: false,
+ formSubmissions: true
+ }
+ end
- describe "plausible-web.js" do
- @example_config %{
- installation_type: :manual,
- track_404_pages: true,
- hash_based_routing: true,
- file_downloads: false,
- outbound_links: false,
- pageview_props: false,
- tagged_events: true,
- revenue_tracking: false,
- form_submissions: true
- }
-
- test "can calculate config" do
- site = new_site()
- tracker_script_configuration = create_config(site)
-
- assert PlausibleWeb.Tracker.plausible_main_config(tracker_script_configuration) == %{
- domain: site.domain,
- endpoint: "#{PlausibleWeb.Endpoint.url()}/api/event",
- outboundLinks: false,
- fileDownloads: false,
- formSubmissions: true
- }
- end
-
+ describe "get_or_create_tracker_script_configuration!/1 and update_script_configuration!/3" do
test "can create config with params" do
site = new_site()
@@ -116,12 +117,14 @@ defmodule PlausibleWeb.TrackerTest do
)
end
end
+ end
+ describe "build_script/1" do
test "can turn config into a script tag" do
site = new_site()
tracker_script_configuration = create_config(site)
- script_tag = PlausibleWeb.Tracker.plausible_main_script_tag(tracker_script_configuration)
+ script_tag = PlausibleWeb.Tracker.build_script(tracker_script_configuration)
assert script_tag =~
~s(={endpoint:"#{PlausibleWeb.Endpoint.url()}/api/event",domain:"#{site.domain}",formSubmissions:!0})
@@ -130,21 +133,84 @@ defmodule PlausibleWeb.TrackerTest do
test "script tag escapes problematic characters as expected" do
site = new_site(domain: "naughty domain &amp;<> \"\'\nfoo ")
tracker_script_configuration = create_config(site)
- script_tag = PlausibleWeb.Tracker.plausible_main_script_tag(tracker_script_configuration)
+ script_tag = PlausibleWeb.Tracker.build_script(tracker_script_configuration)
assert script_tag =~ ~s(domain:"naughty domain &amp;<> \\"'\\nfoo ")
end
+ end
- def create_config(site) do
- Tracker.update_script_configuration!(
- site,
- Map.put(@example_config, :site_id, site.id),
- :installation
- )
+ describe "get_plausible_main_script/1" do
+ setup %{test: test} do
+ site = new_site()
+ tracker_script_configuration = create_config(site)
- Plausible.Repo.one(
- from(c in TrackerScriptConfiguration, where: c.site_id == ^site.id, preload: [:site])
- )
+ opts = start_test_cache(test)
+
+ {:ok, %{opts: opts, site: site, tracker_script_configuration: tracker_script_configuration}}
end
+
+ test "does 0 database queries on non-existing config", %{opts: opts} do
+ query_count = attach_query_counter()
+ script = Tracker.get_plausible_main_script("non-existing-id", opts)
+ assert is_nil(script)
+ assert query_count.() == 0
+ end
+
+ test "does minimal database queries on existing config", %{
+ opts: opts,
+ tracker_script_configuration: tracker_script_configuration
+ } do
+ query_count = attach_query_counter()
+ script = Tracker.get_plausible_main_script(tracker_script_configuration.id, opts)
+ assert is_binary(script)
+
+ on_ee do
+ assert query_count.() == 1
+ else
+ # On self-hosted, we have a pre-warmed cache for the script
+ assert query_count.() == 0
+ end
+ end
+ end
+
+ def create_config(site) do
+ Tracker.update_script_configuration!(
+ site,
+ Map.put(@example_config, :site_id, site.id),
+ :installation
+ )
+
+ Plausible.Repo.one(
+ from(c in TrackerScriptConfiguration, where: c.site_id == ^site.id, preload: [:site])
+ )
+ end
+
+ defp start_test_cache(cache_name) do
+ opts = [cache_name: cache_name, force?: true]
+
+ %{start: {m, f, a}} = TrackerScriptCache.child_spec(cache_name: cache_name)
+ {:ok, _} = apply(m, f, a)
+ TrackerScriptCache.refresh_all(opts)
+
+ opts
+ end
+
+ defp attach_query_counter() do
+ query_count = :counters.new(1, [])
+
+ :telemetry.attach(
+ "test-query-counter",
+ [:plausible, :repo, :query],
+ fn _event, _measurements, _metadata, _config ->
+ :counters.add(query_count, 1, 1)
+ end,
+ %{}
+ )
+
+ on_exit(fn ->
+ :telemetry.detach("test-query-counter")
+ end)
+
+ fn -> :counters.get(query_count, 1) end
end
end