Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Fix: application startup, event timestamps (#2479)

Adam Rutkowski467317f24e68
diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex
index d9d015a84e..1a6c36d5d7 100644
--- a/lib/plausible/application.ex
+++ b/lib/plausible/application.ex
@@ -21,12 +21,12 @@ defmodule Plausible.Application do
Supervisor.child_spec({Cachex, name: :sessions, limit: nil, stats: true},
id: :cachex_sessions
),
- Plausible.PromEx,
{Plausible.Site.Cache, []},
{Plausible.Site.Cache.Warmer.All, []},
{Plausible.Site.Cache.Warmer.RecentlyUpdated, []},
PlausibleWeb.Endpoint,
- {Oban, Application.get_env(:plausible, Oban)}
+ {Oban, Application.get_env(:plausible, Oban)},
+ Plausible.PromEx
]
opts = [strategy: :one_for_one, name: Plausible.Supervisor]
diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.ex
index 3264affdde..695a1d2e1d 100644
--- a/lib/plausible/ingestion/request.ex
+++ b/lib/plausible/ingestion/request.ex
@@ -22,8 +22,7 @@ defmodule Plausible.Ingestion.Request do
field :props, :map
field :query_params, :map
- field :timestamp, :naive_datetime,
- default: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second)
+ field :timestamp, :naive_datetime
end
@type t() :: %__MODULE__{}
@@ -33,7 +32,13 @@ defmodule Plausible.Ingestion.Request do
Builds and initially validates %Plausible.Ingestion.Request{} struct from %Plug.Conn{}.
"""
def build(%Plug.Conn{} = conn) do
- changeset = Changeset.change(%__MODULE__{})
+ changeset =
+ %__MODULE__{}
+ |> Changeset.change()
+ |> Changeset.put_change(
+ :timestamp,
+ NaiveDateTime.utc_now()
+ )
case parse_body(conn) do
{:ok, request_body} ->
diff --git a/lib/plausible/site/cache.ex b/lib/plausible/site/cache.ex
index 8069d655f1..f85a780d4f 100644
--- a/lib/plausible/site/cache.ex
+++ b/lib/plausible/site/cache.ex
@@ -55,6 +55,7 @@ defmodule Plausible.Site.Cache do
@type t() :: Site.t()
+ @spec name() :: atom()
def name(), do: @cache_name
@spec child_spec(Keyword.t()) :: Supervisor.child_spec()
@@ -68,6 +69,22 @@ defmodule Plausible.Site.Cache do
)
end
+ @doc """
+ Ensures the cache has non-zero size unless no sites exist.
+ Useful for orchestrating app startup to prevent the service
+ going up asynchronously with an empty cache.
+ """
+ @spec ready?(atom()) :: boolean
+ def ready?(cache_name \\ @cache_name) do
+ case size(cache_name) do
+ n when n > 0 ->
+ true
+
+ 0 ->
+ Plausible.Repo.aggregate(Site, :count) == 0
+ end
+ end
+
@spec refresh_all(Keyword.t()) :: :ok
def refresh_all(opts \\ []) do
sites_by_domain_query =
diff --git a/lib/plausible_web/controllers/api/external_controller.ex b/lib/plausible_web/controllers/api/external_controller.ex
index f81b4944a2..29e5dd9063 100644
--- a/lib/plausible_web/controllers/api/external_controller.ex
+++ b/lib/plausible_web/controllers/api/external_controller.ex
@@ -60,16 +60,22 @@ defmodule PlausibleWeb.Api.ExternalController do
e -> "error: #{inspect(e)}"
end
+ sites_cache_health =
+ if postgres_health == "ok" and Plausible.Site.Cache.ready?() do
+ "ok"
+ end
+
status =
- case {postgres_health, clickhouse_health} do
- {"ok", "ok"} -> 200
+ case {postgres_health, clickhouse_health, sites_cache_health} do
+ {"ok", "ok", "ok"} -> 200
_ -> 500
end
put_status(conn, status)
|> json(%{
postgres: postgres_health,
- clickhouse: clickhouse_health
+ clickhouse: clickhouse_health,
+ sites_cache: sites_cache_health
})
end
diff --git a/test/plausible/ingestion/request_test.exs b/test/plausible/ingestion/request_test.exs
index 10d529ac2c..320f5aa8d7 100644
--- a/test/plausible/ingestion/request_test.exs
+++ b/test/plausible/ingestion/request_test.exs
@@ -43,6 +43,24 @@ defmodule Plausible.Ingestion.RequestTest do
assert request.props == %{}
end
+ test "requests include moving timestamp" do
+ payload = %{
+ name: "pageview",
+ url: "http://dummy.site"
+ }
+
+ conn = build_conn(:post, "/api/events", payload)
+
+ assert {:ok, request1} = Request.build(conn)
+ :timer.sleep(1500)
+ assert {:ok, request2} = Request.build(conn)
+
+ ts1 = request1.timestamp
+ ts2 = request2.timestamp
+
+ assert NaiveDateTime.compare(ts1, ts2) == :lt
+ end
+
test "request can be built with domain" do
payload = %{
name: "pageview",
diff --git a/test/plausible/site/cache_test.exs b/test/plausible/site/cache_test.exs
index a6a34ca13a..8db8c00f19 100644
--- a/test/plausible/site/cache_test.exs
+++ b/test/plausible/site/cache_test.exs
@@ -53,6 +53,24 @@ defmodule Plausible.Site.CacheTest do
refute Cache.get("site3.example.com", cache_name: test, force?: true)
end
+ test "cache is ready when no sites exist in the db", %{test: test} do
+ {:ok, _} = start_test_cache(test)
+ assert Cache.ready?(test)
+ end
+
+ test "cache is not ready when sites exist in the db but cache needs refresh", %{test: test} do
+ {:ok, _} = start_test_cache(test)
+ insert(:site)
+ refute Cache.ready?(test)
+ end
+
+ test "cache is ready when refreshed", %{test: test} do
+ {:ok, _} = start_test_cache(test)
+ insert(:site)
+ :ok = Cache.refresh_all(cache_name: test)
+ assert Cache.ready?(test)
+ end
+
test "cache exposes hit rate", %{test: test} do
{:ok, _} = start_test_cache(test)
diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs
index eae881be33..cab75db5c2 100644
--- a/test/plausible_web/controllers/api/external_controller_test.exs
+++ b/test/plausible_web/controllers/api/external_controller_test.exs
@@ -62,7 +62,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert conn.status == 400
end
- test "can send to multiple dashboards by listing multiple domains", %{
+ test "can send to multiple dashboards by listing multiple domains - same timestamp", %{
conn: conn,
domain: domain1
} do
@@ -82,8 +82,34 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
|> post("/api/event", params)
assert response(conn, 202) == "ok"
- assert get_event(domain1)
- assert get_event(domain2)
+ assert e1 = get_event(domain1)
+ assert e2 = get_event(domain2)
+
+ assert NaiveDateTime.compare(e1.timestamp, e2.timestamp) == :eq
+ end
+
+ test "timestamps differ when two events sent in a row", %{conn: conn, domain: domain} do
+ params = %{
+ domain: domain,
+ name: "pageview",
+ url: "http://gigride.live/",
+ referrer: "http://m.facebook.com/",
+ screen_width: 1440
+ }
+
+ conn
+ |> put_req_header("user-agent", @user_agent)
+ |> post("/api/event", params)
+
+ :timer.sleep(1500)
+
+ conn
+ |> put_req_header("user-agent", @user_agent)
+ |> post("/api/event", params)
+
+ [e1, e2] = get_events(domain)
+
+ assert NaiveDateTime.compare(e1.timestamp, e2.timestamp) == :gt
end
test "www. is stripped from domain", %{conn: conn, domain: domain} do
@@ -1120,7 +1146,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
test "returns 200 OK", %{conn: conn} do
conn = get(conn, "/api/health")
- assert conn.status == 200
+ assert payload = json_response(conn, 200)
+ assert payload["postgres"] == "ok"
+ assert payload["clickhouse"] == "ok"
+ assert payload["sites_cache"] == "ok"
end
end