ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
feat(analytics): data model + public collector (Phase 1)
Juan Denis713e0e811c82
diff --git a/backend/tests/test_analytics_extension.py b/backend/tests/test_analytics_extension.pyindex ea31a871..43e0f51a 100644--- a/backend/tests/test_analytics_extension.py+++ b/backend/tests/test_analytics_extension.py@@ -30,16 +30,30 @@ def _load_ext():assert plugin_service._ensure_builtin_backend_importable(SLUG), (f'builtin extension backend not importable from {EXT_DIR}')mods = {}- for name in ('config', 'lifecycle', 'analytics'):+ for name in ('config', 'models', 'ingest_service', 'rollup_service',+ 'lifecycle', 'jobs', 'analytics'):mods[name] = importlib.import_module(f'app.plugins.{SLUG}.{name}')return mods_M = _load_ext()config_mod = _M['config']+models_mod = _M['models']+ingest_mod = _M['ingest_service']+rollup_mod = _M['rollup_service']lifecycle_mod = _M['lifecycle']+jobs_mod = _M['jobs']bp_mod = _M['analytics']+AnalyticsSite = models_mod.AnalyticsSite+AnalyticsEvent = models_mod.AnalyticsEvent+AnalyticsDaily = models_mod.AnalyticsDaily+AnalyticsLogCursor = models_mod.AnalyticsLogCursor++# A realistic desktop UA so happy-path hits aren't bot/empty-UA filtered.+UA_CHROME = ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '+ '(KHTML, like Gecko) Chrome/120.0 Safari/537.36')+# --------------------------------------------------------------------------- ## helpers / fixtures@@ -86,6 +100,16 @@ def _auth(user):return {'Authorization': f'Bearer {create_access_token(identity=user.id)}'}[email protected](autouse=True)+def _clean_ingest_state():+ """Reset the process-global ingest buffer + rate limiter between tests."""+ ingest_mod.reset_buffer()+ ingest_mod.reset_rate_limits()+ yield+ ingest_mod.reset_buffer()+ ingest_mod.reset_rate_limits()++@pytest.fixturedef analytics_client(app):if 'analytics' not in app.blueprints:@@ -93,6 +117,28 @@ def analytics_client(app):return app.test_client()+def _mk_site(**kwargs):+ from app import db+ kwargs.setdefault('name', 'Test Site')+ kwargs.setdefault('hostnames', 'example.com')+ site = AnalyticsSite(**kwargs)+ db.session.add(site)+ db.session.commit()+ return site+++def _collect(client, site_key, path='/home', ua=UA_CHROME, extra_headers=None,+ body=None, **payload):+ headers = {'User-Agent': ua, 'Content-Type': 'text/plain'}+ if extra_headers:+ headers.update(extra_headers)+ if body is None:+ data = {'k': site_key, 'p': path}+ data.update(payload)+ body = json.dumps(data)+ return client.post('/api/v1/analytics/collect', data=body, headers=headers)++# --------------------------------------------------------------------------- ## manifest# --------------------------------------------------------------------------- #@@ -211,3 +257,236 @@ def test_ping_route(analytics_client):resp = analytics_client.get('/api/v1/analytics/ping')assert resp.status_code == 200assert resp.get_json()['plugin'] == SLUG+++# --------------------------------------------------------------------------- #+# models + purge+# --------------------------------------------------------------------------- #+def test_models_register_and_tables_exist(app):+ from app import db+ from sqlalchemy import inspect+ assert set(models_mod.register(db)) == {+ AnalyticsSite, AnalyticsEvent, AnalyticsDaily, AnalyticsLogCursor}+ tables = [t for t in inspect(db.engine).get_table_names()+ if t.startswith('ext_serverkit_analytics')]+ assert 'ext_serverkit_analytics_sites' in tables+ assert 'ext_serverkit_analytics_events' in tables+ assert 'ext_serverkit_analytics_daily' in tables+ assert 'ext_serverkit_analytics_log_cursors' in tables+++def test_purge_drops_only_prefixed_tables(app):+ from app import db+ from app.services import extension_lifecycle+ from sqlalchemy import inspect+ from types import SimpleNamespace+ dropped = extension_lifecycle.purge_models(SimpleNamespace(slug=SLUG))+ assert dropped >= 4+ remaining = [t for t in inspect(db.engine).get_table_names()+ if t.startswith('ext_serverkit_analytics')]+ assert remaining == []+ assert 'users' in inspect(db.engine).get_table_names() # core survives+ db.create_all() # restore for later tests in this module+++def test_site_key_is_generated_and_unique(app):+ a = _mk_site(name='A')+ b = _mk_site(name='B')+ assert a.site_key and b.site_key and a.site_key != b.site_key+++# --------------------------------------------------------------------------- #+# ingest units: bot filter, UA parse, visitor hash, helpers+# --------------------------------------------------------------------------- #+def test_is_bot_matches_common_agents():+ assert ingest_mod.is_bot('curl/7.68.0')+ assert ingest_mod.is_bot('Googlebot/2.1')+ assert ingest_mod.is_bot('python-requests/2.31')+ assert not ingest_mod.is_bot(UA_CHROME)+++def test_parse_ua_classifies():+ browser, os_family, device = ingest_mod.parse_ua(UA_CHROME)+ assert browser == 'Chrome'+ assert os_family == 'Windows'+ assert device == 'desktop'+ iphone = ('Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) '+ 'AppleWebKit/605 (KHTML, like Gecko) Version/17 Mobile/15E Safari/604')+ b2, os2, d2 = ingest_mod.parse_ua(iphone)+ assert os2 == 'iOS' and d2 == 'mobile'+++def test_visitor_hash_stable_and_scoped(app):+ h1 = ingest_mod.visitor_hash(1, '1.2.3.4', UA_CHROME)+ h2 = ingest_mod.visitor_hash(1, '1.2.3.4', UA_CHROME)+ h_other_site = ingest_mod.visitor_hash(2, '1.2.3.4', UA_CHROME)+ h_other_ip = ingest_mod.visitor_hash(1, '9.9.9.9', UA_CHROME)+ assert h1 == h2 # deterministic within the day+ assert len(h1) == 32+ assert h1 != h_other_site # scoped by site+ assert h1 != h_other_ip # scoped by ip+ assert '1.2.3.4' not in h1 # raw ip never leaks into the digest+++def test_referrer_host_and_clean_path():+ assert ingest_mod.referrer_host('https://google.com/search?q=x') == 'google.com'+ assert ingest_mod.referrer_host('') is None+ assert ingest_mod._clean_path('https://example.com/a/b?x=1#frag') == '/a/b'+ assert ingest_mod._clean_path('') == '/'+ assert ingest_mod._clean_path('noslash') == '/noslash'+++# --------------------------------------------------------------------------- #+# collector route+# --------------------------------------------------------------------------- #+def test_collect_happy_path_buffers_and_flushes(app, analytics_client):+ site = _mk_site()+ resp = _collect(analytics_client, site.site_key, path='/home', r='https://google.com/')+ assert resp.status_code == 204+ assert ingest_mod.buffer_size() == 1+ written = ingest_mod.flush_buffer(app)+ assert written == 1+ ev = AnalyticsEvent.query.filter_by(site_id=site.id).one()+ assert ev.url_path == '/home'+ assert ev.referrer_host == 'google.com'+ assert ev.ua_family == 'Chrome'+ assert ev.source == 'js'+ assert ev.visitor_hash and len(ev.visitor_hash) == 32+++def test_collect_unknown_key_404(app, analytics_client):+ resp = _collect(analytics_client, 'nope-not-a-real-key')+ assert resp.status_code == 404+ assert ingest_mod.buffer_size() == 0+++def test_collect_disabled_site_403(app, analytics_client):+ site = _mk_site(enabled=False)+ resp = _collect(analytics_client, site.site_key)+ assert resp.status_code == 403+++def test_collect_missing_key_400(app, analytics_client):+ resp = analytics_client.post('/api/v1/analytics/collect',+ data=json.dumps({'p': '/x'}),+ headers={'User-Agent': UA_CHROME})+ assert resp.status_code == 400+++def test_collect_malformed_json_400(app, analytics_client):+ resp = analytics_client.post('/api/v1/analytics/collect',+ data='{not json',+ headers={'User-Agent': UA_CHROME})+ assert resp.status_code == 400+++def test_collect_oversized_body_413(app, analytics_client):+ site = _mk_site()+ big = json.dumps({'k': site.site_key, 'p': '/x', 'pad': 'A' * 9000})+ resp = _collect(analytics_client, site.site_key, body=big)+ assert resp.status_code == 413+++def test_collect_bot_ua_dropped(app, analytics_client):+ site = _mk_site()+ resp = _collect(analytics_client, site.site_key, ua='curl/7.68.0')+ assert resp.status_code == 204+ assert ingest_mod.buffer_size() == 0 # accepted but not stored+++def test_collect_empty_ua_dropped(app, analytics_client):+ site = _mk_site()+ resp = _collect(analytics_client, site.site_key, ua='')+ assert resp.status_code == 204+ assert ingest_mod.buffer_size() == 0+++def test_collect_honors_dnt(app, analytics_client):+ site = _mk_site()+ resp = _collect(analytics_client, site.site_key, extra_headers={'DNT': '1'})+ assert resp.status_code == 204+ assert ingest_mod.buffer_size() == 0+++def test_collect_per_site_dnt_override(app, analytics_client):+ # Global honor_dnt default True; a site opting OUT still records a DNT hit.+ site = _mk_site(honor_dnt=False)+ resp = _collect(analytics_client, site.site_key, extra_headers={'DNT': '1'})+ assert resp.status_code == 204+ assert ingest_mod.buffer_size() == 1+++def test_collect_rate_limited_429(app, analytics_client):+ _mk_plugin_row(config={'collect_rate_per_min': 2})+ site = _mk_site()+ assert _collect(analytics_client, site.site_key).status_code == 204+ assert _collect(analytics_client, site.site_key).status_code == 204+ assert _collect(analytics_client, site.site_key).status_code == 429+++def test_collect_origin_allowlist_enforced(app, analytics_client):+ site = _mk_site(allowed_origins='https://example.com')+ ok = _collect(analytics_client, site.site_key,+ extra_headers={'Origin': 'https://example.com'})+ assert ok.status_code == 204+ assert ok.headers.get('Access-Control-Allow-Origin') == 'https://example.com'+ bad = _collect(analytics_client, site.site_key,+ extra_headers={'Origin': 'https://evil.test'})+ assert bad.status_code == 403+++def test_collect_reflects_origin_by_default(app, analytics_client):+ site = _mk_site() # no allowlist => reflect+ resp = _collect(analytics_client, site.site_key,+ extra_headers={'Origin': 'https://anything.test'})+ assert resp.headers.get('Access-Control-Allow-Origin') == 'https://anything.test'+++def test_collect_options_preflight(app, analytics_client):+ resp = analytics_client.options('/api/v1/analytics/collect',+ headers={'Origin': 'https://example.com'})+ assert resp.status_code == 204+ assert resp.headers.get('Access-Control-Allow-Methods') == 'POST, OPTIONS'+++def test_collect_stores_query_only_when_enabled(app, analytics_client):+ _mk_plugin_row(config={'store_query_strings': True})+ site = _mk_site()+ _collect(analytics_client, site.site_key, path='/search', q='term=hats')+ ingest_mod.flush_buffer(app)+ ev = AnalyticsEvent.query.filter_by(site_id=site.id).one()+ assert ev.url_query == 'term=hats'+++def test_tracker_js_served_public(app, analytics_client):+ resp = analytics_client.get('/api/v1/analytics/tracker.js')+ assert resp.status_code == 200+ assert 'javascript' in resp.headers['Content-Type']+ assert resp.headers.get('Access-Control-Allow-Origin') == '*'+ assert resp.headers.get('Cache-Control') == 'no-cache'+++# --------------------------------------------------------------------------- #+# retention prune+# --------------------------------------------------------------------------- #+def test_retention_prune_deletes_old_events(app):+ from app import db+ from datetime import datetime, timedelta+ site = _mk_site()+ old = AnalyticsEvent(site_id=site.id, type='pageview', url_path='/old',+ ts=datetime.utcnow() - timedelta(days=99), source='js')+ fresh = AnalyticsEvent(site_id=site.id, type='pageview', url_path='/new',+ ts=datetime.utcnow(), source='js')+ db.session.add_all([old, fresh])+ db.session.commit()+ _mk_plugin_row(config={'raw_retention_days': 30})+ result = rollup_mod.run_retention_prune()+ assert result['deleted_events'] == 1+ remaining = [e.url_path for e in AnalyticsEvent.query.filter_by(site_id=site.id)]+ assert remaining == ['/new']+++def test_retention_job_handler_never_raises(app):+ _mk_plugin_row()+ out = jobs_mod.retention_prune(job=None)+ assert 'deleted_events' in out or 'error' in outdiff --git a/builtin-extensions/serverkit-analytics/backend/analytics.py b/builtin-extensions/serverkit-analytics/backend/analytics.pyindex 295ab0f3..ba0e1ba0 100644--- a/builtin-extensions/serverkit-analytics/backend/analytics.py+++ b/builtin-extensions/serverkit-analytics/backend/analytics.py@@ -4,26 +4,186 @@ Mounted under ``/api/v1/analytics`` via the manifest ``url_prefix``. Thinrouting layer over the extension services:* ``ingest_service`` — validation, bot filter, visitor hashing, buffer+flush-* ``report_service`` — rollup-backed dashboard queries-* ``log_ingest_service`` — access-log parsing (apache/nginx combined)-* ``wp_integration`` / ``nginx_integration`` — one-click tracker injection+* ``report_service`` — rollup-backed dashboard queries (Phase 2)+* ``log_ingest_service`` — access-log parsing (Phase 6)+* ``wp_integration`` / ``nginx_integration`` — one-click tracker injection (Ph5)Auth split mirrors serverkit-tramo / serverkit-k8s: report reads need viewer,-site mutations need admin. The TWO exceptions are the tracker surface — the-public ``POST /collect`` and ``GET /tracker.js`` — which carry NO JWT because a-tracked site's visitors have none. They authenticate per-site via the site_key-(not a JWT) and are protected by rate limiting, a body-size cap, and bot-filtering. The plugin-disabled 503 guard (attached by plugin_service) still runs-on every route, so disabling the extension stops collection for free.+site mutations need admin. The public tracker surface — ``POST /collect`` and+``GET /tracker.js`` — carries NO JWT because a tracked site's visitors have+none. It authenticates per-site via the ``site_key`` (not a JWT) and is+protected by a body-size cap, per-site origin allowlist, an in-process token+bucket, and bot/DNT filtering. The plugin-disabled 503 guard (attached by+plugin_service) still runs on every route, so disabling the extension stops+collection for free; the tracker fails silently on any non-2xx."""+import jsonimport logging+import os-from flask import Blueprint, jsonify+from flask import Blueprint, current_app, jsonify, request++from app.utils.client_ip import get_client_ip++from . import ingest_service+from .config import cfg_bool+from .ingest_service import MAX_BODY_BYTES, referrer_host+from .models import AnalyticsSitelogger = logging.getLogger(__name__)analytics_bp = Blueprint('analytics', __name__)+_TRACKER_CACHE = {'js': None}+++# --------------------------------------------------------------------------- #+# helpers+# --------------------------------------------------------------------------- #+def _plain(body='', status=204):+ """A bare Response (empty by default) used for accept-and-drop / success."""+ return current_app.make_response((body, status))+++def _origin_allowed(origin, allowed):+ ohost = referrer_host(origin) or origin+ for entry in allowed:+ ehost = referrer_host(entry) or entry+ if origin == entry or ohost == ehost:+ return True+ return False+++def _acao(site, origin):+ """The Access-Control-Allow-Origin value for this site+origin, or None."""+ if not origin:+ return None+ allowed = site.allowed_origin_list()+ if allowed:+ return origin if _origin_allowed(origin, allowed) else None+ return origin # default: reflect the origin (token already validated)+++def _cors(resp, site, origin):+ acao = _acao(site, origin)+ if acao:+ resp.headers['Access-Control-Allow-Origin'] = acao+ resp.headers['Vary'] = 'Origin'+ return resp+++def _preflight(origin):+ resp = _plain('', 204)+ if origin:+ resp.headers['Access-Control-Allow-Origin'] = origin+ resp.headers['Vary'] = 'Origin'+ resp.headers['Access-Control-Allow-Methods'] = 'POST, OPTIONS'+ resp.headers['Access-Control-Allow-Headers'] = 'Content-Type'+ resp.headers['Access-Control-Max-Age'] = '86400'+ return resp+++# --------------------------------------------------------------------------- #+# public collector (NO auth — per-site token + rate limit + bot filter)+# --------------------------------------------------------------------------- #+@analytics_bp.route('/collect', methods=['POST', 'OPTIONS'])+def collect():+ origin = request.headers.get('Origin')+ if request.method == 'OPTIONS':+ return _preflight(origin)++ # Body-size cap (declared length first, then actual bytes).+ if request.content_length and request.content_length > MAX_BODY_BYTES:+ return jsonify({'error': 'payload too large'}), 413+ raw = request.get_data(cache=False) or b''+ if len(raw) > MAX_BODY_BYTES:+ return jsonify({'error': 'payload too large'}), 413++ # Beacons send text/plain (a CORS-safelisted content-type) to avoid a+ # preflight, so parse the body ourselves regardless of Content-Type.+ try:+ payload = json.loads(raw.decode('utf-8', 'ignore') or '{}')+ except (ValueError, TypeError):+ return jsonify({'error': 'invalid payload'}), 400+ if not isinstance(payload, dict):+ return jsonify({'error': 'invalid payload'}), 400++ site_key = payload.get('k') or payload.get('site_key')+ if not site_key or not isinstance(site_key, str):+ return jsonify({'error': 'missing site_key'}), 400++ site = AnalyticsSite.query.filter_by(site_key=site_key).first()+ if not site:+ return jsonify({'error': 'unknown site'}), 404+ if not site.enabled:+ return jsonify({'error': 'site disabled'}), 403++ # Per-site origin allowlist (empty list => reflect after token check).+ allowed = site.allowed_origin_list()+ if allowed and origin and not _origin_allowed(origin, allowed):+ return jsonify({'error': 'origin not allowed'}), 403++ ip = get_client_ip()+ ua = request.headers.get('User-Agent', '') or ''++ # Rate limit (per site_key AND per IP).+ if not ingest_service.rate_ok(site_key, ip):+ resp = jsonify({'error': 'rate limited'})+ resp.status_code = 429+ return _cors(resp, site, origin)++ # Do Not Track — accepted but not stored (server double-checks the tracker).+ honor_dnt = site.honor_dnt if site.honor_dnt is not None else cfg_bool('honor_dnt')+ if honor_dnt and request.headers.get('DNT') == '1':+ return _cors(_plain(), site, origin)++ # Bot filter: known bots and empty-UA JS hits are silently dropped (204) so+ # we never reveal filtering to an abuser.+ if not ua or ingest_service.is_bot(ua):+ return _cors(_plain(), site, origin)++ event = ingest_service.build_event(site, payload, ip, ua)+ if event is None:+ return _cors(_plain(), site, origin)++ ingest_service.record_event(event)+ ingest_service.ensure_flush_thread(current_app._get_current_object())+ return _cors(_plain(), site, origin)+++# --------------------------------------------------------------------------- #+# public tracker script+# --------------------------------------------------------------------------- #+def _load_tracker_js():+ """Return the tracker JS body. Prefers a checked-in minified artifact+ (Phase 3 build output) and falls back to a tiny placeholder."""+ if _TRACKER_CACHE['js'] is not None:+ return _TRACKER_CACHE['js']+ here = os.path.dirname(os.path.abspath(__file__))+ for rel in (os.path.join('tracker', 'sk.min.js'),+ os.path.join('tracker', 'sk.js')):+ path = os.path.join(here, rel)+ try:+ with open(path, encoding='utf-8') as f:+ _TRACKER_CACHE['js'] = f.read()+ return _TRACKER_CACHE['js']+ except OSError:+ continue+ _TRACKER_CACHE['js'] = (+ '/* serverkit-analytics tracker (placeholder; real build in Phase 3) */\n'+ '(function(){})();\n'+ )+ return _TRACKER_CACHE['js']+++@analytics_bp.route('/tracker.js', methods=['GET'])+def tracker_js():+ resp = current_app.make_response(_load_tracker_js())+ resp.headers['Content-Type'] = 'application/javascript; charset=utf-8'+ resp.headers['Cache-Control'] = 'no-cache'+ resp.headers['Access-Control-Allow-Origin'] = '*' # a public static script+ return resp+@analytics_bp.route('/ping', methods=['GET'])def ping():diff --git a/builtin-extensions/serverkit-analytics/backend/ingest_service.py b/builtin-extensions/serverkit-analytics/backend/ingest_service.pynew file mode 100644index 00000000..79f6ffa4--- /dev/null+++ b/builtin-extensions/serverkit-analytics/backend/ingest_service.py@@ -0,0 +1,358 @@+"""Ingestion for the serverkit-analytics collector.++The public ``POST /collect`` route is a thin wrapper; the real work lives here:++* **Visitor hashing** — cookieless identity is ``HMAC(daily_salt, site|ip|ua)``.+ The salt is generated in-process, rotates at UTC midnight, and is NEVER+ persisted. Raw IPs and UAs are hashed and discarded; only the digest is+ stored. A panel restart mid-day rotates the salt early (a documented,+ acceptable cookieless imprecision).+* **Bot filtering** — UA denylist + empty-UA rejection for JS-source hits.+* **Rate limiting** — an in-process token bucket per site_key and per client IP+ (single-worker safe, same constraint as the login throttle and the api+ self-telemetry buffer).+* **Buffering** — events accumulate in an in-memory buffer and a daemon thread+ batch-inserts them every ``buffer_flush_seconds`` (or early at ``buffer_max``),+ mirroring ``app/middleware/api_analytics.py``. Under TESTING the thread is not+ started; tests call :func:`flush_buffer` directly.++Client IP always comes from the plan-48 trusted-proxy helper+(``app.utils.client_ip.get_client_ip``), never ``request.remote_addr`` raw, so a+reverse proxy doesn't turn every visitor into the nginx box.+"""+import hashlib+import hmac+import logging+import re+import secrets+import threading+import time+from datetime import datetime, timezone+from urllib.parse import urlsplit++from .config import cfg_bool, cfg_int++logger = logging.getLogger(__name__)++# Hard body cap for a collect payload (bytes). Larger => 413.+MAX_BODY_BYTES = 8192++# --------------------------------------------------------------------------- #+# Daily-rotating salt (in-memory, never persisted)+# --------------------------------------------------------------------------- #+_salt_lock = threading.Lock()+_salt_day = None # 'YYYY-MM-DD' (UTC) the current salt belongs to+_salt_value = None # bytes+++def _today_utc():+ return datetime.now(timezone.utc).strftime('%Y-%m-%d')+++def _current_salt():+ """Return today's salt, rotating (and discarding yesterday's) on day change."""+ global _salt_day, _salt_value+ day = _today_utc()+ with _salt_lock:+ if _salt_day != day or _salt_value is None:+ _salt_day = day+ _salt_value = secrets.token_bytes(32)+ return _salt_value+++def visitor_hash(site_id, ip, ua):+ """Cookieless visitor id: HMAC(daily_salt, site|ip|ua), hex-truncated to 32."""+ msg = f'{site_id}|{ip or ""}|{ua or ""}'.encode('utf-8', 'ignore')+ digest = hmac.new(_current_salt(), msg, hashlib.sha256).hexdigest()+ return digest[:32]+++# --------------------------------------------------------------------------- #+# Bot filtering + lightweight UA parsing (no heavy dependency)+# --------------------------------------------------------------------------- #+_BOT_RE = re.compile(+ r'(bot|crawl|spider|slurp|curl|wget|python-requests|python-urllib|'+ r'httpclient|headless|phantomjs|puppeteer|playwright|scrapy|libwww|'+ r'go-http-client|okhttp|axios|node-fetch|facebookexternalhit|'+ r'pingdom|uptimerobot|monitor|semrush|ahrefs|mj12bot|dotbot)',+ re.IGNORECASE,+)++_OS_PATTERNS = [+ ('Windows', re.compile(r'Windows NT', re.I)),+ ('iOS', re.compile(r'iPhone|iPad|iPod', re.I)),+ ('Android', re.compile(r'Android', re.I)),+ ('macOS', re.compile(r'Mac OS X|Macintosh', re.I)),+ ('Linux', re.compile(r'Linux', re.I)),+ ('Chrome OS', re.compile(r'CrOS', re.I)),+]+# Order matters: Edge/OPR/Samsung before Chrome, Chrome before Safari.+_BROWSER_PATTERNS = [+ ('Edge', re.compile(r'Edg[/A ]', re.I)),+ ('Opera', re.compile(r'OPR/|Opera', re.I)),+ ('Samsung Internet', re.compile(r'SamsungBrowser', re.I)),+ ('Firefox', re.compile(r'Firefox/', re.I)),+ ('Chrome', re.compile(r'Chrome/|CriOS', re.I)),+ ('Safari', re.compile(r'Safari/', re.I)),+]+_MOBILE_RE = re.compile(r'Mobile|iPhone|Android.*Mobile', re.I)+_TABLET_RE = re.compile(r'iPad|Tablet|Android(?!.*Mobile)', re.I)+++def is_bot(ua):+ """True for known bot/automation user agents."""+ if not ua:+ return False # emptiness handled separately (js-source empty UA rejected)+ return bool(_BOT_RE.search(ua))+++def parse_ua(ua):+ """Return (browser_family, os_family, device_class) from a UA string.++ Deliberately tiny and heuristic — v1 accepts imperfection (no ua-parser+ dependency). Bots are classed device='bot'.+ """+ if not ua:+ return (None, None, None)+ if is_bot(ua):+ return ('Bot', None, 'bot')+ browser = next((name for name, rx in _BROWSER_PATTERNS if rx.search(ua)), 'Other')+ os_family = next((name for name, rx in _OS_PATTERNS if rx.search(ua)), 'Other')+ if _TABLET_RE.search(ua):+ device = 'tablet'+ elif _MOBILE_RE.search(ua):+ device = 'mobile'+ else:+ device = 'desktop'+ return (browser, os_family, device)+++# --------------------------------------------------------------------------- #+# In-process token-bucket rate limiter (per site_key + per IP)+# --------------------------------------------------------------------------- #+_rl_lock = threading.Lock()+_rl_buckets = {} # key -> [tokens, last_refill_epoch]+_rl_last_sweep = 0.0+++def _sweep_buckets(now):+ """Drop idle buckets so the map can't grow unbounded."""+ global _rl_last_sweep+ if now - _rl_last_sweep < 300:+ return+ _rl_last_sweep = now+ stale = [k for k, (_, last) in _rl_buckets.items() if now - last > 600]+ for k in stale:+ _rl_buckets.pop(k, None)+++def _allow_one(key, rate_per_min, now):+ """Token bucket: capacity == rate_per_min, refill rate == rate/60 per sec."""+ capacity = float(rate_per_min)+ refill = capacity / 60.0+ tokens, last = _rl_buckets.get(key, (capacity, now))+ tokens = min(capacity, tokens + (now - last) * refill)+ if tokens < 1.0:+ _rl_buckets[key] = (tokens, now)+ return False+ _rl_buckets[key] = (tokens - 1.0, now)+ return True+++def rate_ok(site_key, ip):+ """Return True if this hit is under the per-key AND per-IP limits."""+ rate = cfg_int('collect_rate_per_min', minimum=1)+ now = time.monotonic()+ with _rl_lock:+ _sweep_buckets(now)+ ok_key = _allow_one(f'k:{site_key}', rate, now)+ ok_ip = _allow_one(f'i:{ip or "?"}', rate, now)+ return ok_key and ok_ip+++def reset_rate_limits():+ """Test/shutdown helper."""+ with _rl_lock:+ _rl_buckets.clear()+++# --------------------------------------------------------------------------- #+# Buffer + flush+# --------------------------------------------------------------------------- #+_buffer = []+_buffer_lock = threading.Lock()+_flush_started = False+_flush_lock = threading.Lock()+# Safety cap so a stalled flusher can't OOM the process on a traffic spike.+_HARD_BUFFER_CAP = 50000+++def buffer_size():+ with _buffer_lock:+ return len(_buffer)+++def reset_buffer():+ """Test/shutdown helper — drop any un-flushed events."""+ with _buffer_lock:+ _buffer.clear()+++def record_event(entry):+ """Append a built event dict to the in-memory buffer (drops oldest on+ overflow so memory stays bounded)."""+ with _buffer_lock:+ _buffer.append(entry)+ if len(_buffer) > _HARD_BUFFER_CAP:+ drop = len(_buffer) - _HARD_BUFFER_CAP+ del _buffer[:drop]+ logger.warning('analytics buffer overflow; dropped %s oldest events', drop)+++def flush_buffer(app):+ """Batch-insert buffered events. Returns the number written. Best-effort."""+ with _buffer_lock:+ if not _buffer:+ return 0+ entries = _buffer[:]+ _buffer.clear()+ from app import db+ from .models import AnalyticsEvent+ written = 0+ try:+ with app.app_context():+ for entry in entries:+ db.session.add(AnalyticsEvent(**entry))+ written += 1+ db.session.commit()+ except Exception as e: # noqa: BLE001+ logger.warning('analytics flush failed (%s events lost): %s', len(entries), e)+ try:+ db.session.rollback()+ except Exception: # noqa: BLE001+ pass+ return 0+ return written+++def _flush_loop(app):+ """Daemon loop: flush on the configured interval OR early at buffer_max."""+ elapsed = 0+ while True:+ time.sleep(1)+ elapsed += 1+ interval = cfg_int('buffer_flush_seconds', minimum=1, maximum=3600)+ maxsize = cfg_int('buffer_max', minimum=1)+ if elapsed >= interval or buffer_size() >= maxsize:+ elapsed = 0+ try:+ flush_buffer(app)+ except Exception as e: # noqa: BLE001+ logger.warning('analytics flush loop error: %s', e)+++def ensure_flush_thread(app):+ """Start the background flush thread once (skipped under TESTING — tests+ flush synchronously)."""+ global _flush_started+ if app.config.get('TESTING') or app.config.get('ENV') == 'testing':+ return+ with _flush_lock:+ if _flush_started:+ return+ t = threading.Thread(target=_flush_loop, args=(app,),+ daemon=True, name='analytics-flush')+ t.start()+ _flush_started = True+++# --------------------------------------------------------------------------- #+# Payload validation + event build+# --------------------------------------------------------------------------- #+_VALID_TYPES = {'pageview', 'event', 'outlink', 'download'}+_SCREEN_CLASSES = {'xs', 'sm', 'md', 'lg', 'xl'}+++def _clean_path(raw):+ """Normalize a URL path: strip origin, cap length, default '/'."""+ if not raw:+ return '/'+ raw = str(raw)+ # Accept a full URL or a bare path; keep only the path component.+ if '://' in raw:+ raw = urlsplit(raw).path or '/'+ raw = raw.split('#', 1)[0]+ if not raw.startswith('/'):+ raw = '/' + raw+ return raw[:512]+++def referrer_host(raw):+ """Extract a bare host from a referrer URL (None if same-page/empty)."""+ if not raw:+ return None+ try:+ host = urlsplit(str(raw)).hostname+ except ValueError:+ return None+ return host[:255] if host else None+++def build_event(site, payload, ip, ua):+ """Build a persisted-event dict from a validated collect payload.++ Returns ``None`` if the payload is unusable. Does NOT enforce bot/DNT/rate+ policy — that's the collector route's job (so it can pick the right status).+ """+ etype = str(payload.get('t') or payload.get('type') or 'pageview').lower()+ if etype not in _VALID_TYPES:+ etype = 'event'++ raw_path = payload.get('p') if payload.get('p') is not None else payload.get('path')+ path = _clean_path(raw_path)++ query = None+ if cfg_bool('store_query_strings'):+ q = payload.get('q') or payload.get('query')+ if q:+ query = str(q)[:512]+ elif raw_path and '?' in str(raw_path):+ query = str(raw_path).split('?', 1)[1][:512]++ browser, os_family, device = parse_ua(ua)++ screen = payload.get('s') or payload.get('screen')+ screen = screen if screen in _SCREEN_CLASSES else None++ lang = payload.get('l') or payload.get('lang')+ lang = str(lang)[:16] if lang else None++ load_ms = payload.get('ms') or payload.get('load_ms')+ try:+ load_ms = int(load_ms) if load_ms is not None else None+ if load_ms is not None and (load_ms < 0 or load_ms > 3600000):+ load_ms = None+ except (TypeError, ValueError):+ load_ms = None++ name = payload.get('n') or payload.get('name')+ name = str(name)[:128] if name else None++ return {+ 'site_id': site.id,+ 'ts': datetime.utcnow(),+ 'type': etype,+ 'event_name': name,+ 'url_path': path,+ 'url_query': query,+ 'referrer_host': referrer_host(payload.get('r') or payload.get('referrer')),+ 'visitor_hash': visitor_hash(site.id, ip, ua),+ 'ua_family': browser,+ 'os_family': os_family,+ 'device_class': device,+ 'screen_class': screen,+ 'lang': lang,+ 'country': None, # filled by geo in Phase 6 when a GeoLite2 DB is present+ 'source': 'js',+ 'load_ms': load_ms,+ }diff --git a/builtin-extensions/serverkit-analytics/backend/models.py b/builtin-extensions/serverkit-analytics/backend/models.pynew file mode 100644index 00000000..e2aeb541--- /dev/null+++ b/builtin-extensions/serverkit-analytics/backend/models.py@@ -0,0 +1,242 @@+"""Data models for the serverkit-analytics (Web Analytics) extension.++Four tables, all namespaced ``ext_serverkit_analytics_*`` (dash -> underscore)+so ``--purge`` on uninstall drops exactly these:++* :class:`AnalyticsSite` — a tracked site: name, hostnames, the public+ ``site_key`` baked into the tracker snippet (NOT a JWT), per-site settings.+* :class:`AnalyticsEvent` — raw, append-only pageview/event rows (short+ retention). Visitor identity is a daily-rotating salted hash; raw IP/UA are+ never stored.+* :class:`AnalyticsDaily` — the rollup time series (long retention), one row+ per (site, date, dimension, value).+* :class:`AnalyticsLogCursor` — per-site incremental server-log offsets so log+ ingestion is restart-safe.++Registration mirrors the serverkit-tramo / serverkit-k8s pattern: importing this+module defines the tables on the shared metadata as a side effect; the manifest's+``models: "models:register"`` then calls :func:`register` (a passthrough) and the+platform runs ``db.create_all()``.+"""+import json+import secrets+from datetime import datetime++from app import db+++def generate_site_key():+ """A URL-safe public token embedded in the tracker snippet (not a secret in+ the JWT sense — it only names a site; abuse is handled by rate limits + bot+ filtering + origin allowlist)."""+ return secrets.token_urlsafe(16)+++class AnalyticsSite(db.Model):+ __tablename__ = 'ext_serverkit_analytics_sites'++ id = db.Column(db.Integer, primary_key=True)+ name = db.Column(db.String(255), nullable=False)+ # Newline/comma-normalized list of hostnames this site serves.+ hostnames = db.Column(db.Text, nullable=True)+ # Public, URL-safe token baked into the snippet. Unique + indexed for the+ # collector's hot-path lookup.+ site_key = db.Column(db.String(64), unique=True, nullable=False, index=True,+ default=generate_site_key)++ # manual | wordpress | template | nginx+ created_from = db.Column(db.String(32), nullable=False, default='manual')+ # Optional link to a managed app/site id (WordPress site id, service id, ...).+ app_id = db.Column(db.Integer, nullable=True, index=True)++ # Per-site override of the global honor_dnt config. NULL = inherit global.+ honor_dnt = db.Column(db.Boolean, nullable=True)+ # Comma/newline list of allowed origins. Empty/NULL = reflect the request+ # origin after the site_key check (the default per the plan).+ allowed_origins = db.Column(db.Text, nullable=True)++ enabled = db.Column(db.Boolean, nullable=False, default=True)+ # Extensible per-site flags as JSON: wp_injected, nginx_injected,+ # log_ingestion, log_source_kind, log_source_ref, container_name, vhost_path…+ settings = db.Column(db.Text, nullable=True)++ created_at = db.Column(db.DateTime, default=datetime.utcnow)+ updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)++ # ---- helpers ----+ def get_settings(self):+ if not self.settings:+ return {}+ try:+ return json.loads(self.settings)+ except (ValueError, TypeError):+ return {}++ def set_settings(self, value):+ self.settings = json.dumps(value or {})++ def update_settings(self, **changes):+ cur = self.get_settings()+ cur.update(changes)+ self.set_settings(cur)++ def hostname_list(self):+ return _split_list(self.hostnames)++ def allowed_origin_list(self):+ return _split_list(self.allowed_origins)++ def to_dict(self, include_key=True):+ out = {+ 'id': self.id,+ 'name': self.name,+ 'hostnames': self.hostname_list(),+ 'created_from': self.created_from,+ 'app_id': self.app_id,+ 'honor_dnt': self.honor_dnt,+ 'allowed_origins': self.allowed_origin_list(),+ 'enabled': self.enabled,+ 'settings': self.get_settings(),+ 'created_at': self.created_at.isoformat() if self.created_at else None,+ 'updated_at': self.updated_at.isoformat() if self.updated_at else None,+ }+ if include_key:+ out['site_key'] = self.site_key+ return out+++class AnalyticsEvent(db.Model):+ __tablename__ = 'ext_serverkit_analytics_events'++ id = db.Column(db.Integer, primary_key=True)+ site_id = db.Column(db.Integer, nullable=False, index=True)+ ts = db.Column(db.DateTime, nullable=False, default=datetime.utcnow, index=True)++ # pageview | event | outlink | download+ type = db.Column(db.String(16), nullable=False, default='pageview')+ event_name = db.Column(db.String(128), nullable=True)++ url_path = db.Column(db.String(512), nullable=True)+ url_query = db.Column(db.String(512), nullable=True) # only if store_query_strings+ referrer_host = db.Column(db.String(255), nullable=True)++ # HMAC(daily_salt, site_id|ip|ua) — the cookieless visitor id. Never a raw IP.+ visitor_hash = db.Column(db.String(64), nullable=True, index=True)++ ua_family = db.Column(db.String(64), nullable=True)+ os_family = db.Column(db.String(64), nullable=True)+ device_class = db.Column(db.String(16), nullable=True) # desktop|mobile|tablet|bot+ screen_class = db.Column(db.String(16), nullable=True) # xs|sm|md|lg|xl+ lang = db.Column(db.String(16), nullable=True)+ country = db.Column(db.String(2), nullable=True) # ISO-3166-1 alpha-2, optional++ source = db.Column(db.String(8), nullable=False, default='js') # js | log+ load_ms = db.Column(db.Integer, nullable=True)++ __table_args__ = (+ db.Index('ix_ext_analytics_event_site_ts', 'site_id', 'ts'),+ )++ def to_dict(self):+ return {+ 'id': self.id,+ 'site_id': self.site_id,+ 'ts': self.ts.isoformat() if self.ts else None,+ 'type': self.type,+ 'event_name': self.event_name,+ 'url_path': self.url_path,+ 'referrer_host': self.referrer_host,+ 'ua_family': self.ua_family,+ 'os_family': self.os_family,+ 'device_class': self.device_class,+ 'country': self.country,+ 'source': self.source,+ 'load_ms': self.load_ms,+ }+++class AnalyticsDaily(db.Model):+ __tablename__ = 'ext_serverkit_analytics_daily'++ id = db.Column(db.Integer, primary_key=True)+ site_id = db.Column(db.Integer, nullable=False, index=True)+ date = db.Column(db.Date, nullable=False, index=True)++ # overall | page | referrer | device | browser | os | country+ dim_type = db.Column(db.String(16), nullable=False)+ dim_value = db.Column(db.String(512), nullable=False, default='')++ visitors = db.Column(db.Integer, nullable=False, default=0)+ visits = db.Column(db.Integer, nullable=False, default=0)+ pageviews = db.Column(db.Integer, nullable=False, default=0)+ bounces = db.Column(db.Integer, nullable=False, default=0)+ avg_load_ms = db.Column(db.Float, nullable=True)++ __table_args__ = (+ db.UniqueConstraint('site_id', 'date', 'dim_type', 'dim_value',+ name='uq_ext_analytics_daily_dim'),+ )++ def to_dict(self):+ return {+ 'id': self.id,+ 'site_id': self.site_id,+ 'date': self.date.isoformat() if self.date else None,+ 'dim_type': self.dim_type,+ 'dim_value': self.dim_value,+ 'visitors': self.visitors,+ 'visits': self.visits,+ 'pageviews': self.pageviews,+ 'bounces': self.bounces,+ 'avg_load_ms': self.avg_load_ms,+ }+++class AnalyticsLogCursor(db.Model):+ __tablename__ = 'ext_serverkit_analytics_log_cursors'++ id = db.Column(db.Integer, primary_key=True)+ site_id = db.Column(db.Integer, nullable=False, index=True)+ source_kind = db.Column(db.String(16), nullable=False, default='docker') # docker|file+ source_ref = db.Column(db.String(512), nullable=False) # container name or file path+ inode = db.Column(db.String(64), nullable=True)+ byte_offset = db.Column(db.BigInteger, nullable=False, default=0)+ last_ts = db.Column(db.DateTime, nullable=True)+ updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)++ __table_args__ = (+ db.UniqueConstraint('site_id', 'source_ref',+ name='uq_ext_analytics_log_cursor'),+ )++ def to_dict(self):+ return {+ 'id': self.id,+ 'site_id': self.site_id,+ 'source_kind': self.source_kind,+ 'source_ref': self.source_ref,+ 'byte_offset': self.byte_offset,+ 'last_ts': self.last_ts.isoformat() if self.last_ts else None,+ }+++def _split_list(text):+ """Split a comma/newline/whitespace-separated string into a clean list."""+ if not text:+ return []+ parts = []+ for chunk in str(text).replace('\r', '\n').replace(',', '\n').split('\n'):+ item = chunk.strip()+ if item:+ parts.append(item)+ return parts+++def register(db): # noqa: A002 - signature dictated by the platform (fn(db))+ """Passthrough required by the manifest ``models: "models:register"``.++ Importing this module already defined the tables on ``db.metadata``; the+ platform runs ``db.create_all()``. Returning the classes is convenient for+ tests.+ """+ return [AnalyticsSite, AnalyticsEvent, AnalyticsDaily, AnalyticsLogCursor]diff --git a/builtin-extensions/serverkit-analytics/backend/rollup_service.py b/builtin-extensions/serverkit-analytics/backend/rollup_service.pynew file mode 100644index 00000000..67437575--- /dev/null+++ b/builtin-extensions/serverkit-analytics/backend/rollup_service.py@@ -0,0 +1,51 @@+"""Rollups + retention for serverkit-analytics.++* :func:`run_retention_prune` — drop raw ``event`` rows older than+ ``raw_retention_days`` and ``daily`` rollups older than+ ``rollup_retention_months``. Implemented in Phase 1 (operates on the event+ table that exists from Phase 1; the daily table is pruned harmlessly while+ empty until Phase 2 populates it).+* :func:`run_rollup` — aggregate raw events into the daily table. Implemented in+ Phase 2.++Both are called by the scheduled jobs in ``jobs.py`` and must be safe to call+repeatedly (idempotent) and outside a request context (they open their own+session via the shared ``db``).+"""+import logging+from datetime import datetime, timedelta++logger = logging.getLogger(__name__)+++def run_retention_prune():+ """Delete raw events + rollups past their retention windows. Idempotent."""+ from app import db+ from .config import cfg_int+ from .models import AnalyticsEvent, AnalyticsDaily++ raw_days = cfg_int('raw_retention_days', minimum=1, maximum=3650)+ rollup_months = cfg_int('rollup_retention_months', minimum=1, maximum=120)++ event_cutoff = datetime.utcnow() - timedelta(days=raw_days)+ daily_cutoff = (datetime.utcnow() - timedelta(days=rollup_months * 31)).date()++ deleted_events = (AnalyticsEvent.query+ .filter(AnalyticsEvent.ts < event_cutoff)+ .delete(synchronize_session=False))+ deleted_daily = (AnalyticsDaily.query+ .filter(AnalyticsDaily.date < daily_cutoff)+ .delete(synchronize_session=False))+ db.session.commit()++ return {+ 'deleted_events': int(deleted_events or 0),+ 'deleted_daily': int(deleted_daily or 0),+ 'event_cutoff': event_cutoff.isoformat(),+ 'daily_cutoff': daily_cutoff.isoformat(),+ }+++def run_rollup():+ """Aggregate raw events into the daily rollup table. Filled in Phase 2."""+ return {'skipped': True, 'reason': 'rollup implemented in Phase 2'}