Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

feat(analytics): data model + public collector (Phase 1)

Juan Denis713e0e811c82
diff --git a/backend/tests/test_analytics_extension.py b/backend/tests/test_analytics_extension.py
index ea31a871..43e0f51a 100644
--- a/backend/tests/test_analytics_extension.py
+++ b/backend/tests/test_analytics_extension.py
@@ -30,16 +30,30 @@ def _load_ext():
assert plugin_service._ensure_builtin_backend_importable(SLUG), (
f'builtin extension backend not importable from {EXT_DIR}')
mods = {}
- for name in ('config', 'lifecycle', 'analytics'):
+ for name in ('config', 'models', 'ingest_service', 'rollup_service',
+ 'lifecycle', 'jobs', 'analytics'):
mods[name] = importlib.import_module(f'app.plugins.{SLUG}.{name}')
return mods
_M = _load_ext()
config_mod = _M['config']
+models_mod = _M['models']
+ingest_mod = _M['ingest_service']
+rollup_mod = _M['rollup_service']
lifecycle_mod = _M['lifecycle']
+jobs_mod = _M['jobs']
bp_mod = _M['analytics']
+AnalyticsSite = models_mod.AnalyticsSite
+AnalyticsEvent = models_mod.AnalyticsEvent
+AnalyticsDaily = models_mod.AnalyticsDaily
+AnalyticsLogCursor = models_mod.AnalyticsLogCursor
+
+# A realistic desktop UA so happy-path hits aren't bot/empty-UA filtered.
+UA_CHROME = ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+ '(KHTML, like Gecko) Chrome/120.0 Safari/537.36')
+
# --------------------------------------------------------------------------- #
# helpers / fixtures
@@ -86,6 +100,16 @@ def _auth(user):
return {'Authorization': f'Bearer {create_access_token(identity=user.id)}'}
[email protected](autouse=True)
+def _clean_ingest_state():
+ """Reset the process-global ingest buffer + rate limiter between tests."""
+ ingest_mod.reset_buffer()
+ ingest_mod.reset_rate_limits()
+ yield
+ ingest_mod.reset_buffer()
+ ingest_mod.reset_rate_limits()
+
+
@pytest.fixture
def analytics_client(app):
if 'analytics' not in app.blueprints:
@@ -93,6 +117,28 @@ def analytics_client(app):
return app.test_client()
+def _mk_site(**kwargs):
+ from app import db
+ kwargs.setdefault('name', 'Test Site')
+ kwargs.setdefault('hostnames', 'example.com')
+ site = AnalyticsSite(**kwargs)
+ db.session.add(site)
+ db.session.commit()
+ return site
+
+
+def _collect(client, site_key, path='/home', ua=UA_CHROME, extra_headers=None,
+ body=None, **payload):
+ headers = {'User-Agent': ua, 'Content-Type': 'text/plain'}
+ if extra_headers:
+ headers.update(extra_headers)
+ if body is None:
+ data = {'k': site_key, 'p': path}
+ data.update(payload)
+ body = json.dumps(data)
+ return client.post('/api/v1/analytics/collect', data=body, headers=headers)
+
+
# --------------------------------------------------------------------------- #
# manifest
# --------------------------------------------------------------------------- #
@@ -211,3 +257,236 @@ def test_ping_route(analytics_client):
resp = analytics_client.get('/api/v1/analytics/ping')
assert resp.status_code == 200
assert resp.get_json()['plugin'] == SLUG
+
+
+# --------------------------------------------------------------------------- #
+# models + purge
+# --------------------------------------------------------------------------- #
+def test_models_register_and_tables_exist(app):
+ from app import db
+ from sqlalchemy import inspect
+ assert set(models_mod.register(db)) == {
+ AnalyticsSite, AnalyticsEvent, AnalyticsDaily, AnalyticsLogCursor}
+ tables = [t for t in inspect(db.engine).get_table_names()
+ if t.startswith('ext_serverkit_analytics')]
+ assert 'ext_serverkit_analytics_sites' in tables
+ assert 'ext_serverkit_analytics_events' in tables
+ assert 'ext_serverkit_analytics_daily' in tables
+ assert 'ext_serverkit_analytics_log_cursors' in tables
+
+
+def test_purge_drops_only_prefixed_tables(app):
+ from app import db
+ from app.services import extension_lifecycle
+ from sqlalchemy import inspect
+ from types import SimpleNamespace
+ dropped = extension_lifecycle.purge_models(SimpleNamespace(slug=SLUG))
+ assert dropped >= 4
+ remaining = [t for t in inspect(db.engine).get_table_names()
+ if t.startswith('ext_serverkit_analytics')]
+ assert remaining == []
+ assert 'users' in inspect(db.engine).get_table_names() # core survives
+ db.create_all() # restore for later tests in this module
+
+
+def test_site_key_is_generated_and_unique(app):
+ a = _mk_site(name='A')
+ b = _mk_site(name='B')
+ assert a.site_key and b.site_key and a.site_key != b.site_key
+
+
+# --------------------------------------------------------------------------- #
+# ingest units: bot filter, UA parse, visitor hash, helpers
+# --------------------------------------------------------------------------- #
+def test_is_bot_matches_common_agents():
+ assert ingest_mod.is_bot('curl/7.68.0')
+ assert ingest_mod.is_bot('Googlebot/2.1')
+ assert ingest_mod.is_bot('python-requests/2.31')
+ assert not ingest_mod.is_bot(UA_CHROME)
+
+
+def test_parse_ua_classifies():
+ browser, os_family, device = ingest_mod.parse_ua(UA_CHROME)
+ assert browser == 'Chrome'
+ assert os_family == 'Windows'
+ assert device == 'desktop'
+ iphone = ('Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) '
+ 'AppleWebKit/605 (KHTML, like Gecko) Version/17 Mobile/15E Safari/604')
+ b2, os2, d2 = ingest_mod.parse_ua(iphone)
+ assert os2 == 'iOS' and d2 == 'mobile'
+
+
+def test_visitor_hash_stable_and_scoped(app):
+ h1 = ingest_mod.visitor_hash(1, '1.2.3.4', UA_CHROME)
+ h2 = ingest_mod.visitor_hash(1, '1.2.3.4', UA_CHROME)
+ h_other_site = ingest_mod.visitor_hash(2, '1.2.3.4', UA_CHROME)
+ h_other_ip = ingest_mod.visitor_hash(1, '9.9.9.9', UA_CHROME)
+ assert h1 == h2 # deterministic within the day
+ assert len(h1) == 32
+ assert h1 != h_other_site # scoped by site
+ assert h1 != h_other_ip # scoped by ip
+ assert '1.2.3.4' not in h1 # raw ip never leaks into the digest
+
+
+def test_referrer_host_and_clean_path():
+ assert ingest_mod.referrer_host('https://google.com/search?q=x') == 'google.com'
+ assert ingest_mod.referrer_host('') is None
+ assert ingest_mod._clean_path('https://example.com/a/b?x=1#frag') == '/a/b'
+ assert ingest_mod._clean_path('') == '/'
+ assert ingest_mod._clean_path('noslash') == '/noslash'
+
+
+# --------------------------------------------------------------------------- #
+# collector route
+# --------------------------------------------------------------------------- #
+def test_collect_happy_path_buffers_and_flushes(app, analytics_client):
+ site = _mk_site()
+ resp = _collect(analytics_client, site.site_key, path='/home', r='https://google.com/')
+ assert resp.status_code == 204
+ assert ingest_mod.buffer_size() == 1
+ written = ingest_mod.flush_buffer(app)
+ assert written == 1
+ ev = AnalyticsEvent.query.filter_by(site_id=site.id).one()
+ assert ev.url_path == '/home'
+ assert ev.referrer_host == 'google.com'
+ assert ev.ua_family == 'Chrome'
+ assert ev.source == 'js'
+ assert ev.visitor_hash and len(ev.visitor_hash) == 32
+
+
+def test_collect_unknown_key_404(app, analytics_client):
+ resp = _collect(analytics_client, 'nope-not-a-real-key')
+ assert resp.status_code == 404
+ assert ingest_mod.buffer_size() == 0
+
+
+def test_collect_disabled_site_403(app, analytics_client):
+ site = _mk_site(enabled=False)
+ resp = _collect(analytics_client, site.site_key)
+ assert resp.status_code == 403
+
+
+def test_collect_missing_key_400(app, analytics_client):
+ resp = analytics_client.post('/api/v1/analytics/collect',
+ data=json.dumps({'p': '/x'}),
+ headers={'User-Agent': UA_CHROME})
+ assert resp.status_code == 400
+
+
+def test_collect_malformed_json_400(app, analytics_client):
+ resp = analytics_client.post('/api/v1/analytics/collect',
+ data='{not json',
+ headers={'User-Agent': UA_CHROME})
+ assert resp.status_code == 400
+
+
+def test_collect_oversized_body_413(app, analytics_client):
+ site = _mk_site()
+ big = json.dumps({'k': site.site_key, 'p': '/x', 'pad': 'A' * 9000})
+ resp = _collect(analytics_client, site.site_key, body=big)
+ assert resp.status_code == 413
+
+
+def test_collect_bot_ua_dropped(app, analytics_client):
+ site = _mk_site()
+ resp = _collect(analytics_client, site.site_key, ua='curl/7.68.0')
+ assert resp.status_code == 204
+ assert ingest_mod.buffer_size() == 0 # accepted but not stored
+
+
+def test_collect_empty_ua_dropped(app, analytics_client):
+ site = _mk_site()
+ resp = _collect(analytics_client, site.site_key, ua='')
+ assert resp.status_code == 204
+ assert ingest_mod.buffer_size() == 0
+
+
+def test_collect_honors_dnt(app, analytics_client):
+ site = _mk_site()
+ resp = _collect(analytics_client, site.site_key, extra_headers={'DNT': '1'})
+ assert resp.status_code == 204
+ assert ingest_mod.buffer_size() == 0
+
+
+def test_collect_per_site_dnt_override(app, analytics_client):
+ # Global honor_dnt default True; a site opting OUT still records a DNT hit.
+ site = _mk_site(honor_dnt=False)
+ resp = _collect(analytics_client, site.site_key, extra_headers={'DNT': '1'})
+ assert resp.status_code == 204
+ assert ingest_mod.buffer_size() == 1
+
+
+def test_collect_rate_limited_429(app, analytics_client):
+ _mk_plugin_row(config={'collect_rate_per_min': 2})
+ site = _mk_site()
+ assert _collect(analytics_client, site.site_key).status_code == 204
+ assert _collect(analytics_client, site.site_key).status_code == 204
+ assert _collect(analytics_client, site.site_key).status_code == 429
+
+
+def test_collect_origin_allowlist_enforced(app, analytics_client):
+ site = _mk_site(allowed_origins='https://example.com')
+ ok = _collect(analytics_client, site.site_key,
+ extra_headers={'Origin': 'https://example.com'})
+ assert ok.status_code == 204
+ assert ok.headers.get('Access-Control-Allow-Origin') == 'https://example.com'
+ bad = _collect(analytics_client, site.site_key,
+ extra_headers={'Origin': 'https://evil.test'})
+ assert bad.status_code == 403
+
+
+def test_collect_reflects_origin_by_default(app, analytics_client):
+ site = _mk_site() # no allowlist => reflect
+ resp = _collect(analytics_client, site.site_key,
+ extra_headers={'Origin': 'https://anything.test'})
+ assert resp.headers.get('Access-Control-Allow-Origin') == 'https://anything.test'
+
+
+def test_collect_options_preflight(app, analytics_client):
+ resp = analytics_client.options('/api/v1/analytics/collect',
+ headers={'Origin': 'https://example.com'})
+ assert resp.status_code == 204
+ assert resp.headers.get('Access-Control-Allow-Methods') == 'POST, OPTIONS'
+
+
+def test_collect_stores_query_only_when_enabled(app, analytics_client):
+ _mk_plugin_row(config={'store_query_strings': True})
+ site = _mk_site()
+ _collect(analytics_client, site.site_key, path='/search', q='term=hats')
+ ingest_mod.flush_buffer(app)
+ ev = AnalyticsEvent.query.filter_by(site_id=site.id).one()
+ assert ev.url_query == 'term=hats'
+
+
+def test_tracker_js_served_public(app, analytics_client):
+ resp = analytics_client.get('/api/v1/analytics/tracker.js')
+ assert resp.status_code == 200
+ assert 'javascript' in resp.headers['Content-Type']
+ assert resp.headers.get('Access-Control-Allow-Origin') == '*'
+ assert resp.headers.get('Cache-Control') == 'no-cache'
+
+
+# --------------------------------------------------------------------------- #
+# retention prune
+# --------------------------------------------------------------------------- #
+def test_retention_prune_deletes_old_events(app):
+ from app import db
+ from datetime import datetime, timedelta
+ site = _mk_site()
+ old = AnalyticsEvent(site_id=site.id, type='pageview', url_path='/old',
+ ts=datetime.utcnow() - timedelta(days=99), source='js')
+ fresh = AnalyticsEvent(site_id=site.id, type='pageview', url_path='/new',
+ ts=datetime.utcnow(), source='js')
+ db.session.add_all([old, fresh])
+ db.session.commit()
+ _mk_plugin_row(config={'raw_retention_days': 30})
+ result = rollup_mod.run_retention_prune()
+ assert result['deleted_events'] == 1
+ remaining = [e.url_path for e in AnalyticsEvent.query.filter_by(site_id=site.id)]
+ assert remaining == ['/new']
+
+
+def test_retention_job_handler_never_raises(app):
+ _mk_plugin_row()
+ out = jobs_mod.retention_prune(job=None)
+ assert 'deleted_events' in out or 'error' in out
diff --git a/builtin-extensions/serverkit-analytics/backend/analytics.py b/builtin-extensions/serverkit-analytics/backend/analytics.py
index 295ab0f3..ba0e1ba0 100644
--- a/builtin-extensions/serverkit-analytics/backend/analytics.py
+++ b/builtin-extensions/serverkit-analytics/backend/analytics.py
@@ -4,26 +4,186 @@ Mounted under ``/api/v1/analytics`` via the manifest ``url_prefix``. Thin
routing layer over the extension services:
* ``ingest_service`` — validation, bot filter, visitor hashing, buffer+flush
-* ``report_service`` — rollup-backed dashboard queries
-* ``log_ingest_service`` — access-log parsing (apache/nginx combined)
-* ``wp_integration`` / ``nginx_integration`` — one-click tracker injection
+* ``report_service`` — rollup-backed dashboard queries (Phase 2)
+* ``log_ingest_service`` — access-log parsing (Phase 6)
+* ``wp_integration`` / ``nginx_integration`` — one-click tracker injection (Ph5)
Auth split mirrors serverkit-tramo / serverkit-k8s: report reads need viewer,
-site mutations need admin. The TWO exceptions are the tracker surface — the
-public ``POST /collect`` and ``GET /tracker.js`` — which carry NO JWT because a
-tracked site's visitors have none. They authenticate per-site via the site_key
-(not a JWT) and are protected by rate limiting, a body-size cap, and bot
-filtering. The plugin-disabled 503 guard (attached by plugin_service) still runs
-on every route, so disabling the extension stops collection for free.
+site mutations need admin. The public tracker surface — ``POST /collect`` and
+``GET /tracker.js`` — carries NO JWT because a tracked site's visitors have
+none. It authenticates per-site via the ``site_key`` (not a JWT) and is
+protected by a body-size cap, per-site origin allowlist, an in-process token
+bucket, and bot/DNT filtering. The plugin-disabled 503 guard (attached by
+plugin_service) still runs on every route, so disabling the extension stops
+collection for free; the tracker fails silently on any non-2xx.
"""
+import json
import logging
+import os
-from flask import Blueprint, jsonify
+from flask import Blueprint, current_app, jsonify, request
+
+from app.utils.client_ip import get_client_ip
+
+from . import ingest_service
+from .config import cfg_bool
+from .ingest_service import MAX_BODY_BYTES, referrer_host
+from .models import AnalyticsSite
logger = logging.getLogger(__name__)
analytics_bp = Blueprint('analytics', __name__)
+_TRACKER_CACHE = {'js': None}
+
+
+# --------------------------------------------------------------------------- #
+# helpers
+# --------------------------------------------------------------------------- #
+def _plain(body='', status=204):
+ """A bare Response (empty by default) used for accept-and-drop / success."""
+ return current_app.make_response((body, status))
+
+
+def _origin_allowed(origin, allowed):
+ ohost = referrer_host(origin) or origin
+ for entry in allowed:
+ ehost = referrer_host(entry) or entry
+ if origin == entry or ohost == ehost:
+ return True
+ return False
+
+
+def _acao(site, origin):
+ """The Access-Control-Allow-Origin value for this site+origin, or None."""
+ if not origin:
+ return None
+ allowed = site.allowed_origin_list()
+ if allowed:
+ return origin if _origin_allowed(origin, allowed) else None
+ return origin # default: reflect the origin (token already validated)
+
+
+def _cors(resp, site, origin):
+ acao = _acao(site, origin)
+ if acao:
+ resp.headers['Access-Control-Allow-Origin'] = acao
+ resp.headers['Vary'] = 'Origin'
+ return resp
+
+
+def _preflight(origin):
+ resp = _plain('', 204)
+ if origin:
+ resp.headers['Access-Control-Allow-Origin'] = origin
+ resp.headers['Vary'] = 'Origin'
+ resp.headers['Access-Control-Allow-Methods'] = 'POST, OPTIONS'
+ resp.headers['Access-Control-Allow-Headers'] = 'Content-Type'
+ resp.headers['Access-Control-Max-Age'] = '86400'
+ return resp
+
+
+# --------------------------------------------------------------------------- #
+# public collector (NO auth — per-site token + rate limit + bot filter)
+# --------------------------------------------------------------------------- #
+@analytics_bp.route('/collect', methods=['POST', 'OPTIONS'])
+def collect():
+ origin = request.headers.get('Origin')
+ if request.method == 'OPTIONS':
+ return _preflight(origin)
+
+ # Body-size cap (declared length first, then actual bytes).
+ if request.content_length and request.content_length > MAX_BODY_BYTES:
+ return jsonify({'error': 'payload too large'}), 413
+ raw = request.get_data(cache=False) or b''
+ if len(raw) > MAX_BODY_BYTES:
+ return jsonify({'error': 'payload too large'}), 413
+
+ # Beacons send text/plain (a CORS-safelisted content-type) to avoid a
+ # preflight, so parse the body ourselves regardless of Content-Type.
+ try:
+ payload = json.loads(raw.decode('utf-8', 'ignore') or '{}')
+ except (ValueError, TypeError):
+ return jsonify({'error': 'invalid payload'}), 400
+ if not isinstance(payload, dict):
+ return jsonify({'error': 'invalid payload'}), 400
+
+ site_key = payload.get('k') or payload.get('site_key')
+ if not site_key or not isinstance(site_key, str):
+ return jsonify({'error': 'missing site_key'}), 400
+
+ site = AnalyticsSite.query.filter_by(site_key=site_key).first()
+ if not site:
+ return jsonify({'error': 'unknown site'}), 404
+ if not site.enabled:
+ return jsonify({'error': 'site disabled'}), 403
+
+ # Per-site origin allowlist (empty list => reflect after token check).
+ allowed = site.allowed_origin_list()
+ if allowed and origin and not _origin_allowed(origin, allowed):
+ return jsonify({'error': 'origin not allowed'}), 403
+
+ ip = get_client_ip()
+ ua = request.headers.get('User-Agent', '') or ''
+
+ # Rate limit (per site_key AND per IP).
+ if not ingest_service.rate_ok(site_key, ip):
+ resp = jsonify({'error': 'rate limited'})
+ resp.status_code = 429
+ return _cors(resp, site, origin)
+
+ # Do Not Track — accepted but not stored (server double-checks the tracker).
+ honor_dnt = site.honor_dnt if site.honor_dnt is not None else cfg_bool('honor_dnt')
+ if honor_dnt and request.headers.get('DNT') == '1':
+ return _cors(_plain(), site, origin)
+
+ # Bot filter: known bots and empty-UA JS hits are silently dropped (204) so
+ # we never reveal filtering to an abuser.
+ if not ua or ingest_service.is_bot(ua):
+ return _cors(_plain(), site, origin)
+
+ event = ingest_service.build_event(site, payload, ip, ua)
+ if event is None:
+ return _cors(_plain(), site, origin)
+
+ ingest_service.record_event(event)
+ ingest_service.ensure_flush_thread(current_app._get_current_object())
+ return _cors(_plain(), site, origin)
+
+
+# --------------------------------------------------------------------------- #
+# public tracker script
+# --------------------------------------------------------------------------- #
+def _load_tracker_js():
+ """Return the tracker JS body. Prefers a checked-in minified artifact
+ (Phase 3 build output) and falls back to a tiny placeholder."""
+ if _TRACKER_CACHE['js'] is not None:
+ return _TRACKER_CACHE['js']
+ here = os.path.dirname(os.path.abspath(__file__))
+ for rel in (os.path.join('tracker', 'sk.min.js'),
+ os.path.join('tracker', 'sk.js')):
+ path = os.path.join(here, rel)
+ try:
+ with open(path, encoding='utf-8') as f:
+ _TRACKER_CACHE['js'] = f.read()
+ return _TRACKER_CACHE['js']
+ except OSError:
+ continue
+ _TRACKER_CACHE['js'] = (
+ '/* serverkit-analytics tracker (placeholder; real build in Phase 3) */\n'
+ '(function(){})();\n'
+ )
+ return _TRACKER_CACHE['js']
+
+
+@analytics_bp.route('/tracker.js', methods=['GET'])
+def tracker_js():
+ resp = current_app.make_response(_load_tracker_js())
+ resp.headers['Content-Type'] = 'application/javascript; charset=utf-8'
+ resp.headers['Cache-Control'] = 'no-cache'
+ resp.headers['Access-Control-Allow-Origin'] = '*' # a public static script
+ return resp
+
@analytics_bp.route('/ping', methods=['GET'])
def ping():
diff --git a/builtin-extensions/serverkit-analytics/backend/ingest_service.py b/builtin-extensions/serverkit-analytics/backend/ingest_service.py
new file mode 100644
index 00000000..79f6ffa4
--- /dev/null
+++ b/builtin-extensions/serverkit-analytics/backend/ingest_service.py
@@ -0,0 +1,358 @@
+"""Ingestion for the serverkit-analytics collector.
+
+The public ``POST /collect`` route is a thin wrapper; the real work lives here:
+
+* **Visitor hashing** — cookieless identity is ``HMAC(daily_salt, site|ip|ua)``.
+ The salt is generated in-process, rotates at UTC midnight, and is NEVER
+ persisted. Raw IPs and UAs are hashed and discarded; only the digest is
+ stored. A panel restart mid-day rotates the salt early (a documented,
+ acceptable cookieless imprecision).
+* **Bot filtering** — UA denylist + empty-UA rejection for JS-source hits.
+* **Rate limiting** — an in-process token bucket per site_key and per client IP
+ (single-worker safe, same constraint as the login throttle and the api
+ self-telemetry buffer).
+* **Buffering** — events accumulate in an in-memory buffer and a daemon thread
+ batch-inserts them every ``buffer_flush_seconds`` (or early at ``buffer_max``),
+ mirroring ``app/middleware/api_analytics.py``. Under TESTING the thread is not
+ started; tests call :func:`flush_buffer` directly.
+
+Client IP always comes from the plan-48 trusted-proxy helper
+(``app.utils.client_ip.get_client_ip``), never ``request.remote_addr`` raw, so a
+reverse proxy doesn't turn every visitor into the nginx box.
+"""
+import hashlib
+import hmac
+import logging
+import re
+import secrets
+import threading
+import time
+from datetime import datetime, timezone
+from urllib.parse import urlsplit
+
+from .config import cfg_bool, cfg_int
+
+logger = logging.getLogger(__name__)
+
+# Hard body cap for a collect payload (bytes). Larger => 413.
+MAX_BODY_BYTES = 8192
+
+# --------------------------------------------------------------------------- #
+# Daily-rotating salt (in-memory, never persisted)
+# --------------------------------------------------------------------------- #
+_salt_lock = threading.Lock()
+_salt_day = None # 'YYYY-MM-DD' (UTC) the current salt belongs to
+_salt_value = None # bytes
+
+
+def _today_utc():
+ return datetime.now(timezone.utc).strftime('%Y-%m-%d')
+
+
+def _current_salt():
+ """Return today's salt, rotating (and discarding yesterday's) on day change."""
+ global _salt_day, _salt_value
+ day = _today_utc()
+ with _salt_lock:
+ if _salt_day != day or _salt_value is None:
+ _salt_day = day
+ _salt_value = secrets.token_bytes(32)
+ return _salt_value
+
+
+def visitor_hash(site_id, ip, ua):
+ """Cookieless visitor id: HMAC(daily_salt, site|ip|ua), hex-truncated to 32."""
+ msg = f'{site_id}|{ip or ""}|{ua or ""}'.encode('utf-8', 'ignore')
+ digest = hmac.new(_current_salt(), msg, hashlib.sha256).hexdigest()
+ return digest[:32]
+
+
+# --------------------------------------------------------------------------- #
+# Bot filtering + lightweight UA parsing (no heavy dependency)
+# --------------------------------------------------------------------------- #
+_BOT_RE = re.compile(
+ r'(bot|crawl|spider|slurp|curl|wget|python-requests|python-urllib|'
+ r'httpclient|headless|phantomjs|puppeteer|playwright|scrapy|libwww|'
+ r'go-http-client|okhttp|axios|node-fetch|facebookexternalhit|'
+ r'pingdom|uptimerobot|monitor|semrush|ahrefs|mj12bot|dotbot)',
+ re.IGNORECASE,
+)
+
+_OS_PATTERNS = [
+ ('Windows', re.compile(r'Windows NT', re.I)),
+ ('iOS', re.compile(r'iPhone|iPad|iPod', re.I)),
+ ('Android', re.compile(r'Android', re.I)),
+ ('macOS', re.compile(r'Mac OS X|Macintosh', re.I)),
+ ('Linux', re.compile(r'Linux', re.I)),
+ ('Chrome OS', re.compile(r'CrOS', re.I)),
+]
+# Order matters: Edge/OPR/Samsung before Chrome, Chrome before Safari.
+_BROWSER_PATTERNS = [
+ ('Edge', re.compile(r'Edg[/A ]', re.I)),
+ ('Opera', re.compile(r'OPR/|Opera', re.I)),
+ ('Samsung Internet', re.compile(r'SamsungBrowser', re.I)),
+ ('Firefox', re.compile(r'Firefox/', re.I)),
+ ('Chrome', re.compile(r'Chrome/|CriOS', re.I)),
+ ('Safari', re.compile(r'Safari/', re.I)),
+]
+_MOBILE_RE = re.compile(r'Mobile|iPhone|Android.*Mobile', re.I)
+_TABLET_RE = re.compile(r'iPad|Tablet|Android(?!.*Mobile)', re.I)
+
+
+def is_bot(ua):
+ """True for known bot/automation user agents."""
+ if not ua:
+ return False # emptiness handled separately (js-source empty UA rejected)
+ return bool(_BOT_RE.search(ua))
+
+
+def parse_ua(ua):
+ """Return (browser_family, os_family, device_class) from a UA string.
+
+ Deliberately tiny and heuristic — v1 accepts imperfection (no ua-parser
+ dependency). Bots are classed device='bot'.
+ """
+ if not ua:
+ return (None, None, None)
+ if is_bot(ua):
+ return ('Bot', None, 'bot')
+ browser = next((name for name, rx in _BROWSER_PATTERNS if rx.search(ua)), 'Other')
+ os_family = next((name for name, rx in _OS_PATTERNS if rx.search(ua)), 'Other')
+ if _TABLET_RE.search(ua):
+ device = 'tablet'
+ elif _MOBILE_RE.search(ua):
+ device = 'mobile'
+ else:
+ device = 'desktop'
+ return (browser, os_family, device)
+
+
+# --------------------------------------------------------------------------- #
+# In-process token-bucket rate limiter (per site_key + per IP)
+# --------------------------------------------------------------------------- #
+_rl_lock = threading.Lock()
+_rl_buckets = {} # key -> [tokens, last_refill_epoch]
+_rl_last_sweep = 0.0
+
+
+def _sweep_buckets(now):
+ """Drop idle buckets so the map can't grow unbounded."""
+ global _rl_last_sweep
+ if now - _rl_last_sweep < 300:
+ return
+ _rl_last_sweep = now
+ stale = [k for k, (_, last) in _rl_buckets.items() if now - last > 600]
+ for k in stale:
+ _rl_buckets.pop(k, None)
+
+
+def _allow_one(key, rate_per_min, now):
+ """Token bucket: capacity == rate_per_min, refill rate == rate/60 per sec."""
+ capacity = float(rate_per_min)
+ refill = capacity / 60.0
+ tokens, last = _rl_buckets.get(key, (capacity, now))
+ tokens = min(capacity, tokens + (now - last) * refill)
+ if tokens < 1.0:
+ _rl_buckets[key] = (tokens, now)
+ return False
+ _rl_buckets[key] = (tokens - 1.0, now)
+ return True
+
+
+def rate_ok(site_key, ip):
+ """Return True if this hit is under the per-key AND per-IP limits."""
+ rate = cfg_int('collect_rate_per_min', minimum=1)
+ now = time.monotonic()
+ with _rl_lock:
+ _sweep_buckets(now)
+ ok_key = _allow_one(f'k:{site_key}', rate, now)
+ ok_ip = _allow_one(f'i:{ip or "?"}', rate, now)
+ return ok_key and ok_ip
+
+
+def reset_rate_limits():
+ """Test/shutdown helper."""
+ with _rl_lock:
+ _rl_buckets.clear()
+
+
+# --------------------------------------------------------------------------- #
+# Buffer + flush
+# --------------------------------------------------------------------------- #
+_buffer = []
+_buffer_lock = threading.Lock()
+_flush_started = False
+_flush_lock = threading.Lock()
+# Safety cap so a stalled flusher can't OOM the process on a traffic spike.
+_HARD_BUFFER_CAP = 50000
+
+
+def buffer_size():
+ with _buffer_lock:
+ return len(_buffer)
+
+
+def reset_buffer():
+ """Test/shutdown helper — drop any un-flushed events."""
+ with _buffer_lock:
+ _buffer.clear()
+
+
+def record_event(entry):
+ """Append a built event dict to the in-memory buffer (drops oldest on
+ overflow so memory stays bounded)."""
+ with _buffer_lock:
+ _buffer.append(entry)
+ if len(_buffer) > _HARD_BUFFER_CAP:
+ drop = len(_buffer) - _HARD_BUFFER_CAP
+ del _buffer[:drop]
+ logger.warning('analytics buffer overflow; dropped %s oldest events', drop)
+
+
+def flush_buffer(app):
+ """Batch-insert buffered events. Returns the number written. Best-effort."""
+ with _buffer_lock:
+ if not _buffer:
+ return 0
+ entries = _buffer[:]
+ _buffer.clear()
+ from app import db
+ from .models import AnalyticsEvent
+ written = 0
+ try:
+ with app.app_context():
+ for entry in entries:
+ db.session.add(AnalyticsEvent(**entry))
+ written += 1
+ db.session.commit()
+ except Exception as e: # noqa: BLE001
+ logger.warning('analytics flush failed (%s events lost): %s', len(entries), e)
+ try:
+ db.session.rollback()
+ except Exception: # noqa: BLE001
+ pass
+ return 0
+ return written
+
+
+def _flush_loop(app):
+ """Daemon loop: flush on the configured interval OR early at buffer_max."""
+ elapsed = 0
+ while True:
+ time.sleep(1)
+ elapsed += 1
+ interval = cfg_int('buffer_flush_seconds', minimum=1, maximum=3600)
+ maxsize = cfg_int('buffer_max', minimum=1)
+ if elapsed >= interval or buffer_size() >= maxsize:
+ elapsed = 0
+ try:
+ flush_buffer(app)
+ except Exception as e: # noqa: BLE001
+ logger.warning('analytics flush loop error: %s', e)
+
+
+def ensure_flush_thread(app):
+ """Start the background flush thread once (skipped under TESTING — tests
+ flush synchronously)."""
+ global _flush_started
+ if app.config.get('TESTING') or app.config.get('ENV') == 'testing':
+ return
+ with _flush_lock:
+ if _flush_started:
+ return
+ t = threading.Thread(target=_flush_loop, args=(app,),
+ daemon=True, name='analytics-flush')
+ t.start()
+ _flush_started = True
+
+
+# --------------------------------------------------------------------------- #
+# Payload validation + event build
+# --------------------------------------------------------------------------- #
+_VALID_TYPES = {'pageview', 'event', 'outlink', 'download'}
+_SCREEN_CLASSES = {'xs', 'sm', 'md', 'lg', 'xl'}
+
+
+def _clean_path(raw):
+ """Normalize a URL path: strip origin, cap length, default '/'."""
+ if not raw:
+ return '/'
+ raw = str(raw)
+ # Accept a full URL or a bare path; keep only the path component.
+ if '://' in raw:
+ raw = urlsplit(raw).path or '/'
+ raw = raw.split('#', 1)[0]
+ if not raw.startswith('/'):
+ raw = '/' + raw
+ return raw[:512]
+
+
+def referrer_host(raw):
+ """Extract a bare host from a referrer URL (None if same-page/empty)."""
+ if not raw:
+ return None
+ try:
+ host = urlsplit(str(raw)).hostname
+ except ValueError:
+ return None
+ return host[:255] if host else None
+
+
+def build_event(site, payload, ip, ua):
+ """Build a persisted-event dict from a validated collect payload.
+
+ Returns ``None`` if the payload is unusable. Does NOT enforce bot/DNT/rate
+ policy — that's the collector route's job (so it can pick the right status).
+ """
+ etype = str(payload.get('t') or payload.get('type') or 'pageview').lower()
+ if etype not in _VALID_TYPES:
+ etype = 'event'
+
+ raw_path = payload.get('p') if payload.get('p') is not None else payload.get('path')
+ path = _clean_path(raw_path)
+
+ query = None
+ if cfg_bool('store_query_strings'):
+ q = payload.get('q') or payload.get('query')
+ if q:
+ query = str(q)[:512]
+ elif raw_path and '?' in str(raw_path):
+ query = str(raw_path).split('?', 1)[1][:512]
+
+ browser, os_family, device = parse_ua(ua)
+
+ screen = payload.get('s') or payload.get('screen')
+ screen = screen if screen in _SCREEN_CLASSES else None
+
+ lang = payload.get('l') or payload.get('lang')
+ lang = str(lang)[:16] if lang else None
+
+ load_ms = payload.get('ms') or payload.get('load_ms')
+ try:
+ load_ms = int(load_ms) if load_ms is not None else None
+ if load_ms is not None and (load_ms < 0 or load_ms > 3600000):
+ load_ms = None
+ except (TypeError, ValueError):
+ load_ms = None
+
+ name = payload.get('n') or payload.get('name')
+ name = str(name)[:128] if name else None
+
+ return {
+ 'site_id': site.id,
+ 'ts': datetime.utcnow(),
+ 'type': etype,
+ 'event_name': name,
+ 'url_path': path,
+ 'url_query': query,
+ 'referrer_host': referrer_host(payload.get('r') or payload.get('referrer')),
+ 'visitor_hash': visitor_hash(site.id, ip, ua),
+ 'ua_family': browser,
+ 'os_family': os_family,
+ 'device_class': device,
+ 'screen_class': screen,
+ 'lang': lang,
+ 'country': None, # filled by geo in Phase 6 when a GeoLite2 DB is present
+ 'source': 'js',
+ 'load_ms': load_ms,
+ }
diff --git a/builtin-extensions/serverkit-analytics/backend/models.py b/builtin-extensions/serverkit-analytics/backend/models.py
new file mode 100644
index 00000000..e2aeb541
--- /dev/null
+++ b/builtin-extensions/serverkit-analytics/backend/models.py
@@ -0,0 +1,242 @@
+"""Data models for the serverkit-analytics (Web Analytics) extension.
+
+Four tables, all namespaced ``ext_serverkit_analytics_*`` (dash -> underscore)
+so ``--purge`` on uninstall drops exactly these:
+
+* :class:`AnalyticsSite` — a tracked site: name, hostnames, the public
+ ``site_key`` baked into the tracker snippet (NOT a JWT), per-site settings.
+* :class:`AnalyticsEvent` — raw, append-only pageview/event rows (short
+ retention). Visitor identity is a daily-rotating salted hash; raw IP/UA are
+ never stored.
+* :class:`AnalyticsDaily` — the rollup time series (long retention), one row
+ per (site, date, dimension, value).
+* :class:`AnalyticsLogCursor` — per-site incremental server-log offsets so log
+ ingestion is restart-safe.
+
+Registration mirrors the serverkit-tramo / serverkit-k8s pattern: importing this
+module defines the tables on the shared metadata as a side effect; the manifest's
+``models: "models:register"`` then calls :func:`register` (a passthrough) and the
+platform runs ``db.create_all()``.
+"""
+import json
+import secrets
+from datetime import datetime
+
+from app import db
+
+
+def generate_site_key():
+ """A URL-safe public token embedded in the tracker snippet (not a secret in
+ the JWT sense — it only names a site; abuse is handled by rate limits + bot
+ filtering + origin allowlist)."""
+ return secrets.token_urlsafe(16)
+
+
+class AnalyticsSite(db.Model):
+ __tablename__ = 'ext_serverkit_analytics_sites'
+
+ id = db.Column(db.Integer, primary_key=True)
+ name = db.Column(db.String(255), nullable=False)
+ # Newline/comma-normalized list of hostnames this site serves.
+ hostnames = db.Column(db.Text, nullable=True)
+ # Public, URL-safe token baked into the snippet. Unique + indexed for the
+ # collector's hot-path lookup.
+ site_key = db.Column(db.String(64), unique=True, nullable=False, index=True,
+ default=generate_site_key)
+
+ # manual | wordpress | template | nginx
+ created_from = db.Column(db.String(32), nullable=False, default='manual')
+ # Optional link to a managed app/site id (WordPress site id, service id, ...).
+ app_id = db.Column(db.Integer, nullable=True, index=True)
+
+ # Per-site override of the global honor_dnt config. NULL = inherit global.
+ honor_dnt = db.Column(db.Boolean, nullable=True)
+ # Comma/newline list of allowed origins. Empty/NULL = reflect the request
+ # origin after the site_key check (the default per the plan).
+ allowed_origins = db.Column(db.Text, nullable=True)
+
+ enabled = db.Column(db.Boolean, nullable=False, default=True)
+ # Extensible per-site flags as JSON: wp_injected, nginx_injected,
+ # log_ingestion, log_source_kind, log_source_ref, container_name, vhost_path…
+ settings = db.Column(db.Text, nullable=True)
+
+ created_at = db.Column(db.DateTime, default=datetime.utcnow)
+ updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+ # ---- helpers ----
+ def get_settings(self):
+ if not self.settings:
+ return {}
+ try:
+ return json.loads(self.settings)
+ except (ValueError, TypeError):
+ return {}
+
+ def set_settings(self, value):
+ self.settings = json.dumps(value or {})
+
+ def update_settings(self, **changes):
+ cur = self.get_settings()
+ cur.update(changes)
+ self.set_settings(cur)
+
+ def hostname_list(self):
+ return _split_list(self.hostnames)
+
+ def allowed_origin_list(self):
+ return _split_list(self.allowed_origins)
+
+ def to_dict(self, include_key=True):
+ out = {
+ 'id': self.id,
+ 'name': self.name,
+ 'hostnames': self.hostname_list(),
+ 'created_from': self.created_from,
+ 'app_id': self.app_id,
+ 'honor_dnt': self.honor_dnt,
+ 'allowed_origins': self.allowed_origin_list(),
+ 'enabled': self.enabled,
+ 'settings': self.get_settings(),
+ 'created_at': self.created_at.isoformat() if self.created_at else None,
+ 'updated_at': self.updated_at.isoformat() if self.updated_at else None,
+ }
+ if include_key:
+ out['site_key'] = self.site_key
+ return out
+
+
+class AnalyticsEvent(db.Model):
+ __tablename__ = 'ext_serverkit_analytics_events'
+
+ id = db.Column(db.Integer, primary_key=True)
+ site_id = db.Column(db.Integer, nullable=False, index=True)
+ ts = db.Column(db.DateTime, nullable=False, default=datetime.utcnow, index=True)
+
+ # pageview | event | outlink | download
+ type = db.Column(db.String(16), nullable=False, default='pageview')
+ event_name = db.Column(db.String(128), nullable=True)
+
+ url_path = db.Column(db.String(512), nullable=True)
+ url_query = db.Column(db.String(512), nullable=True) # only if store_query_strings
+ referrer_host = db.Column(db.String(255), nullable=True)
+
+ # HMAC(daily_salt, site_id|ip|ua) — the cookieless visitor id. Never a raw IP.
+ visitor_hash = db.Column(db.String(64), nullable=True, index=True)
+
+ ua_family = db.Column(db.String(64), nullable=True)
+ os_family = db.Column(db.String(64), nullable=True)
+ device_class = db.Column(db.String(16), nullable=True) # desktop|mobile|tablet|bot
+ screen_class = db.Column(db.String(16), nullable=True) # xs|sm|md|lg|xl
+ lang = db.Column(db.String(16), nullable=True)
+ country = db.Column(db.String(2), nullable=True) # ISO-3166-1 alpha-2, optional
+
+ source = db.Column(db.String(8), nullable=False, default='js') # js | log
+ load_ms = db.Column(db.Integer, nullable=True)
+
+ __table_args__ = (
+ db.Index('ix_ext_analytics_event_site_ts', 'site_id', 'ts'),
+ )
+
+ def to_dict(self):
+ return {
+ 'id': self.id,
+ 'site_id': self.site_id,
+ 'ts': self.ts.isoformat() if self.ts else None,
+ 'type': self.type,
+ 'event_name': self.event_name,
+ 'url_path': self.url_path,
+ 'referrer_host': self.referrer_host,
+ 'ua_family': self.ua_family,
+ 'os_family': self.os_family,
+ 'device_class': self.device_class,
+ 'country': self.country,
+ 'source': self.source,
+ 'load_ms': self.load_ms,
+ }
+
+
+class AnalyticsDaily(db.Model):
+ __tablename__ = 'ext_serverkit_analytics_daily'
+
+ id = db.Column(db.Integer, primary_key=True)
+ site_id = db.Column(db.Integer, nullable=False, index=True)
+ date = db.Column(db.Date, nullable=False, index=True)
+
+ # overall | page | referrer | device | browser | os | country
+ dim_type = db.Column(db.String(16), nullable=False)
+ dim_value = db.Column(db.String(512), nullable=False, default='')
+
+ visitors = db.Column(db.Integer, nullable=False, default=0)
+ visits = db.Column(db.Integer, nullable=False, default=0)
+ pageviews = db.Column(db.Integer, nullable=False, default=0)
+ bounces = db.Column(db.Integer, nullable=False, default=0)
+ avg_load_ms = db.Column(db.Float, nullable=True)
+
+ __table_args__ = (
+ db.UniqueConstraint('site_id', 'date', 'dim_type', 'dim_value',
+ name='uq_ext_analytics_daily_dim'),
+ )
+
+ def to_dict(self):
+ return {
+ 'id': self.id,
+ 'site_id': self.site_id,
+ 'date': self.date.isoformat() if self.date else None,
+ 'dim_type': self.dim_type,
+ 'dim_value': self.dim_value,
+ 'visitors': self.visitors,
+ 'visits': self.visits,
+ 'pageviews': self.pageviews,
+ 'bounces': self.bounces,
+ 'avg_load_ms': self.avg_load_ms,
+ }
+
+
+class AnalyticsLogCursor(db.Model):
+ __tablename__ = 'ext_serverkit_analytics_log_cursors'
+
+ id = db.Column(db.Integer, primary_key=True)
+ site_id = db.Column(db.Integer, nullable=False, index=True)
+ source_kind = db.Column(db.String(16), nullable=False, default='docker') # docker|file
+ source_ref = db.Column(db.String(512), nullable=False) # container name or file path
+ inode = db.Column(db.String(64), nullable=True)
+ byte_offset = db.Column(db.BigInteger, nullable=False, default=0)
+ last_ts = db.Column(db.DateTime, nullable=True)
+ updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+ __table_args__ = (
+ db.UniqueConstraint('site_id', 'source_ref',
+ name='uq_ext_analytics_log_cursor'),
+ )
+
+ def to_dict(self):
+ return {
+ 'id': self.id,
+ 'site_id': self.site_id,
+ 'source_kind': self.source_kind,
+ 'source_ref': self.source_ref,
+ 'byte_offset': self.byte_offset,
+ 'last_ts': self.last_ts.isoformat() if self.last_ts else None,
+ }
+
+
+def _split_list(text):
+ """Split a comma/newline/whitespace-separated string into a clean list."""
+ if not text:
+ return []
+ parts = []
+ for chunk in str(text).replace('\r', '\n').replace(',', '\n').split('\n'):
+ item = chunk.strip()
+ if item:
+ parts.append(item)
+ return parts
+
+
+def register(db): # noqa: A002 - signature dictated by the platform (fn(db))
+ """Passthrough required by the manifest ``models: "models:register"``.
+
+ Importing this module already defined the tables on ``db.metadata``; the
+ platform runs ``db.create_all()``. Returning the classes is convenient for
+ tests.
+ """
+ return [AnalyticsSite, AnalyticsEvent, AnalyticsDaily, AnalyticsLogCursor]
diff --git a/builtin-extensions/serverkit-analytics/backend/rollup_service.py b/builtin-extensions/serverkit-analytics/backend/rollup_service.py
new file mode 100644
index 00000000..67437575
--- /dev/null
+++ b/builtin-extensions/serverkit-analytics/backend/rollup_service.py
@@ -0,0 +1,51 @@
+"""Rollups + retention for serverkit-analytics.
+
+* :func:`run_retention_prune` — drop raw ``event`` rows older than
+ ``raw_retention_days`` and ``daily`` rollups older than
+ ``rollup_retention_months``. Implemented in Phase 1 (operates on the event
+ table that exists from Phase 1; the daily table is pruned harmlessly while
+ empty until Phase 2 populates it).
+* :func:`run_rollup` — aggregate raw events into the daily table. Implemented in
+ Phase 2.
+
+Both are called by the scheduled jobs in ``jobs.py`` and must be safe to call
+repeatedly (idempotent) and outside a request context (they open their own
+session via the shared ``db``).
+"""
+import logging
+from datetime import datetime, timedelta
+
+logger = logging.getLogger(__name__)
+
+
+def run_retention_prune():
+ """Delete raw events + rollups past their retention windows. Idempotent."""
+ from app import db
+ from .config import cfg_int
+ from .models import AnalyticsEvent, AnalyticsDaily
+
+ raw_days = cfg_int('raw_retention_days', minimum=1, maximum=3650)
+ rollup_months = cfg_int('rollup_retention_months', minimum=1, maximum=120)
+
+ event_cutoff = datetime.utcnow() - timedelta(days=raw_days)
+ daily_cutoff = (datetime.utcnow() - timedelta(days=rollup_months * 31)).date()
+
+ deleted_events = (AnalyticsEvent.query
+ .filter(AnalyticsEvent.ts < event_cutoff)
+ .delete(synchronize_session=False))
+ deleted_daily = (AnalyticsDaily.query
+ .filter(AnalyticsDaily.date < daily_cutoff)
+ .delete(synchronize_session=False))
+ db.session.commit()
+
+ return {
+ 'deleted_events': int(deleted_events or 0),
+ 'deleted_daily': int(deleted_daily or 0),
+ 'event_cutoff': event_cutoff.isoformat(),
+ 'daily_cutoff': daily_cutoff.isoformat(),
+ }
+
+
+def run_rollup():
+ """Aggregate raw events into the daily rollup table. Filled in Phase 2."""
+ return {'skipped': True, 'reason': 'rollup implemented in Phase 2'}