Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

feat(analytics): rollups + report API (Phase 2)

Juan Denise0cde1d93ec1
diff --git a/backend/tests/test_analytics_extension.py b/backend/tests/test_analytics_extension.py
index 43e0f51a..de7b77bb 100644
--- a/backend/tests/test_analytics_extension.py
+++ b/backend/tests/test_analytics_extension.py
@@ -31,7 +31,8 @@ def _load_ext():
f'builtin extension backend not importable from {EXT_DIR}')
mods = {}
for name in ('config', 'models', 'ingest_service', 'rollup_service',
- 'lifecycle', 'jobs', 'analytics'):
+ 'report_service', 'site_service', 'lifecycle', 'jobs',
+ 'analytics'):
mods[name] = importlib.import_module(f'app.plugins.{SLUG}.{name}')
return mods
@@ -41,6 +42,8 @@ config_mod = _M['config']
models_mod = _M['models']
ingest_mod = _M['ingest_service']
rollup_mod = _M['rollup_service']
+report_mod = _M['report_service']
+site_mod = _M['site_service']
lifecycle_mod = _M['lifecycle']
jobs_mod = _M['jobs']
bp_mod = _M['analytics']
@@ -490,3 +493,199 @@ def test_retention_job_handler_never_raises(app):
_mk_plugin_row()
out = jobs_mod.retention_prune(job=None)
assert 'deleted_events' in out or 'error' in out
+
+
+# --------------------------------------------------------------------------- #
+# Phase 2: rollup + report API
+# --------------------------------------------------------------------------- #
+def _mk_event(site_id, visitor='v1', path='/home', ref=None, device='desktop',
+ browser='Chrome', os_family='Windows', country=None, load_ms=None,
+ days_ago=0, etype='pageview'):
+ from app import db
+ from datetime import datetime, timedelta
+ ev = AnalyticsEvent(
+ site_id=site_id, type=etype, url_path=path, referrer_host=ref,
+ visitor_hash=visitor, device_class=device, ua_family=browser,
+ os_family=os_family, country=country, load_ms=load_ms, source='js',
+ ts=datetime.utcnow() - timedelta(days=days_ago))
+ db.session.add(ev)
+ db.session.commit()
+ return ev
+
+
+def test_rollup_aggregates_overall(app):
+ site = _mk_site()
+ # Two visitors today: v1 has 2 pageviews (not a bounce), v2 has 1 (a bounce).
+ _mk_event(site.id, visitor='v1', path='/a', load_ms=100)
+ _mk_event(site.id, visitor='v1', path='/b', load_ms=200)
+ _mk_event(site.id, visitor='v2', path='/a', load_ms=300)
+ out = rollup_mod.run_rollup()
+ assert out['rows'] > 0
+ row = AnalyticsDaily.query.filter_by(site_id=site.id, dim_type='overall').one()
+ assert row.visitors == 2
+ assert row.pageviews == 3
+ assert row.bounces == 1 # only v2 had a single pageview
+ assert row.avg_load_ms == 200.0 # (100+200+300)/3
+
+
+def test_rollup_is_idempotent(app):
+ site = _mk_site()
+ _mk_event(site.id, visitor='v1', path='/a')
+ rollup_mod.run_rollup()
+ rollup_mod.run_rollup() # second run must not double-count
+ row = AnalyticsDaily.query.filter_by(site_id=site.id, dim_type='overall').one()
+ assert row.pageviews == 1
+
+
+def test_rollup_dimensions(app):
+ site = _mk_site()
+ _mk_event(site.id, visitor='v1', path='/a', ref='google.com', device='mobile',
+ browser='Firefox', os_family='Android', country='US')
+ _mk_event(site.id, visitor='v2', path='/a', ref='google.com', device='desktop',
+ browser='Chrome', os_family='Windows', country='CA')
+ rollup_mod.run_rollup()
+ page = AnalyticsDaily.query.filter_by(site_id=site.id, dim_type='page',
+ dim_value='/a').one()
+ assert page.pageviews == 2 and page.visitors == 2
+ ref = AnalyticsDaily.query.filter_by(site_id=site.id, dim_type='referrer',
+ dim_value='google.com').one()
+ assert ref.pageviews == 2
+ countries = {r.dim_value for r in AnalyticsDaily.query.filter_by(
+ site_id=site.id, dim_type='country')}
+ assert countries == {'US', 'CA'}
+
+
+def test_report_overview_and_pages(app):
+ from datetime import datetime, timedelta
+ site = _mk_site()
+ _mk_event(site.id, visitor='v1', path='/a')
+ _mk_event(site.id, visitor='v2', path='/a')
+ _mk_event(site.id, visitor='v1', path='/b')
+ rollup_mod.run_rollup()
+ end = datetime.utcnow().date()
+ start = end - timedelta(days=6)
+ ov = report_mod.overview(site.id, start, end)
+ assert ov['totals']['pageviews'] == 3
+ assert ov['totals']['visitors'] == 2
+ assert len(ov['timeseries']) == 7 # zero-filled 7-day window
+ top = {p['value']: p['pageviews'] for p in ov['top_pages']}
+ assert top['/a'] == 2 and top['/b'] == 1
+
+
+def test_report_realtime_reads_raw_events(app):
+ site = _mk_site()
+ _mk_event(site.id, visitor='v1', path='/live')
+ _mk_event(site.id, visitor='v1', path='/live2')
+ _mk_event(site.id, visitor='v2', path='/live')
+ rt = report_mod.realtime(site.id, minutes=30)
+ assert rt['active_visitors'] == 2
+ assert rt['pageviews'] == 3
+ assert rt['recent'][0]['path'] in ('/live', '/live2')
+
+
+def test_parse_range_defaults_and_explicit():
+ from datetime import datetime
+ s, e = report_mod.parse_range({})
+ assert (e - s).days == 6 # default 7d inclusive
+ s2, e2 = report_mod.parse_range({'range': '30d'})
+ assert (e2 - s2).days == 29
+ s3, e3 = report_mod.parse_range({'start': '2026-01-01', 'end': '2026-01-10'})
+ assert s3.isoformat() == '2026-01-01' and e3.isoformat() == '2026-01-10'
+
+
+# --- site CRUD + RBAC ---
+def test_sites_crud_flow(app, analytics_client):
+ admin = _auth(_mk_admin())
+ # create
+ r = analytics_client.post('/api/v1/analytics/sites', headers=admin,
+ json={'name': 'Blog', 'hostnames': ['blog.example.com']})
+ assert r.status_code == 201
+ body = r.get_json()
+ sid = body['id']
+ assert body['site_key']
+ assert body['hostnames'] == ['blog.example.com']
+ # list
+ r = analytics_client.get('/api/v1/analytics/sites', headers=admin)
+ assert any(s['id'] == sid for s in r.get_json()['sites'])
+ # update
+ r = analytics_client.put(f'/api/v1/analytics/sites/{sid}', headers=admin,
+ json={'enabled': False})
+ assert r.get_json()['enabled'] is False
+ # rotate key
+ old_key = body['site_key']
+ r = analytics_client.post(f'/api/v1/analytics/sites/{sid}/rotate-key', headers=admin)
+ assert r.get_json()['site_key'] != old_key
+ # delete
+ r = analytics_client.delete(f'/api/v1/analytics/sites/{sid}', headers=admin)
+ assert r.status_code == 200
+ assert AnalyticsSite.query.get(sid) is None
+
+
+def test_create_site_requires_name_400(app, analytics_client):
+ admin = _auth(_mk_admin())
+ r = analytics_client.post('/api/v1/analytics/sites', headers=admin, json={})
+ assert r.status_code == 400
+
+
+def test_reads_allow_viewer(app, analytics_client):
+ viewer = _auth(_mk_viewer())
+ site = _mk_site()
+ r = analytics_client.get('/api/v1/analytics/sites', headers=viewer)
+ assert r.status_code == 200
+ r = analytics_client.get(f'/api/v1/analytics/sites/{site.id}/overview',
+ headers=viewer)
+ assert r.status_code == 200
+
+
+def test_mutations_deny_viewer_403(app, analytics_client):
+ viewer = _auth(_mk_viewer())
+ r = analytics_client.post('/api/v1/analytics/sites', headers=viewer,
+ json={'name': 'x'})
+ assert r.status_code == 403
+
+
+def test_reports_require_auth_401(app, analytics_client):
+ site = _mk_site()
+ r = analytics_client.get(f'/api/v1/analytics/sites/{site.id}/overview')
+ assert r.status_code in (401, 422) # missing/invalid JWT
+
+
+def test_overview_endpoint_shape(app, analytics_client):
+ admin = _auth(_mk_admin())
+ site = _mk_site()
+ _mk_event(site.id, visitor='v1', path='/a')
+ rollup_mod.run_rollup()
+ r = analytics_client.get(f'/api/v1/analytics/sites/{site.id}/overview?range=30d',
+ headers=admin)
+ assert r.status_code == 200
+ body = r.get_json()
+ assert set(body) >= {'totals', 'timeseries', 'top_pages', 'top_referrers',
+ 'realtime', 'range'}
+
+
+def test_devices_endpoint(app, analytics_client):
+ admin = _auth(_mk_admin())
+ site = _mk_site()
+ _mk_event(site.id, visitor='v1', device='mobile', browser='Safari',
+ os_family='iOS', country='US')
+ rollup_mod.run_rollup()
+ r = analytics_client.get(f'/api/v1/analytics/sites/{site.id}/devices',
+ headers=admin)
+ body = r.get_json()
+ assert body['device'][0]['value'] == 'mobile'
+ assert body['os'][0]['value'] == 'iOS'
+
+
+def test_rollup_endpoint_admin_only(app, analytics_client):
+ viewer = _auth(_mk_viewer())
+ assert analytics_client.post('/api/v1/analytics/rollup',
+ headers=viewer).status_code == 403
+ admin = _auth(_mk_admin())
+ assert analytics_client.post('/api/v1/analytics/rollup',
+ headers=admin).status_code == 200
+
+
+def test_rollup_job_handler_never_raises(app):
+ _mk_plugin_row()
+ out = jobs_mod.rollup(job=None)
+ assert 'rows' in out or 'error' in out or 'sites_dates' in out
diff --git a/builtin-extensions/serverkit-analytics/backend/analytics.py b/builtin-extensions/serverkit-analytics/backend/analytics.py
index ba0e1ba0..234a633d 100644
--- a/builtin-extensions/serverkit-analytics/backend/analytics.py
+++ b/builtin-extensions/serverkit-analytics/backend/analytics.py
@@ -23,9 +23,10 @@ import os
from flask import Blueprint, current_app, jsonify, request
+from app.middleware.rbac import admin_required, viewer_required
from app.utils.client_ip import get_client_ip
-from . import ingest_service
+from . import ingest_service, report_service, rollup_service, site_service
from .config import cfg_bool
from .ingest_service import MAX_BODY_BYTES, referrer_host
from .models import AnalyticsSite
@@ -189,3 +190,139 @@ def tracker_js():
def ping():
"""Unauthenticated liveness probe (also proves the 503 status guard)."""
return jsonify({'ok': True, 'plugin': 'serverkit-analytics'}), 200
+
+
+# --------------------------------------------------------------------------- #
+# site CRUD (JWT: viewer reads, admin mutations)
+# --------------------------------------------------------------------------- #
+def _site_or_404(site_id):
+ site = site_service.get_site(site_id)
+ if not site:
+ return None, (jsonify({'error': 'Site not found'}), 404)
+ return site, None
+
+
+@analytics_bp.route('/sites', methods=['GET'])
+@viewer_required
+def list_sites():
+ return jsonify({'sites': site_service.list_sites()}), 200
+
+
+@analytics_bp.route('/sites', methods=['POST'])
+@admin_required
+def create_site():
+ data = request.get_json(silent=True) or {}
+ try:
+ site = site_service.create_site(data)
+ except ValueError as e:
+ return jsonify({'error': str(e)}), 400
+ return jsonify(site.to_dict()), 201
+
+
+@analytics_bp.route('/sites/<int:site_id>', methods=['GET'])
+@viewer_required
+def get_site(site_id):
+ site, err = _site_or_404(site_id)
+ if err:
+ return err
+ return jsonify(site.to_dict()), 200
+
+
+@analytics_bp.route('/sites/<int:site_id>', methods=['PUT', 'PATCH'])
+@admin_required
+def update_site(site_id):
+ data = request.get_json(silent=True) or {}
+ try:
+ site = site_service.update_site(site_id, data)
+ except ValueError as e:
+ return jsonify({'error': str(e)}), 400
+ if not site:
+ return jsonify({'error': 'Site not found'}), 404
+ return jsonify(site.to_dict()), 200
+
+
+@analytics_bp.route('/sites/<int:site_id>', methods=['DELETE'])
+@admin_required
+def delete_site(site_id):
+ if not site_service.delete_site(site_id):
+ return jsonify({'error': 'Site not found'}), 404
+ return jsonify({'deleted': True}), 200
+
+
+@analytics_bp.route('/sites/<int:site_id>/rotate-key', methods=['POST'])
+@admin_required
+def rotate_key(site_id):
+ site = site_service.rotate_key(site_id)
+ if not site:
+ return jsonify({'error': 'Site not found'}), 404
+ return jsonify(site.to_dict()), 200
+
+
+# --------------------------------------------------------------------------- #
+# report queries (JWT: viewer)
+# --------------------------------------------------------------------------- #
+@analytics_bp.route('/sites/<int:site_id>/overview', methods=['GET'])
+@viewer_required
+def site_overview(site_id):
+ site, err = _site_or_404(site_id)
+ if err:
+ return err
+ start, end = report_service.parse_range(request.args)
+ return jsonify(report_service.overview(site_id, start, end)), 200
+
+
+@analytics_bp.route('/sites/<int:site_id>/timeseries', methods=['GET'])
+@viewer_required
+def site_timeseries(site_id):
+ site, err = _site_or_404(site_id)
+ if err:
+ return err
+ start, end = report_service.parse_range(request.args)
+ return jsonify({'series': report_service.timeseries(site_id, start, end)}), 200
+
+
+@analytics_bp.route('/sites/<int:site_id>/pages', methods=['GET'])
+@viewer_required
+def site_pages(site_id):
+ site, err = _site_or_404(site_id)
+ if err:
+ return err
+ start, end = report_service.parse_range(request.args)
+ return jsonify(report_service.pages(site_id, start, end)), 200
+
+
+@analytics_bp.route('/sites/<int:site_id>/referrers', methods=['GET'])
+@viewer_required
+def site_referrers(site_id):
+ site, err = _site_or_404(site_id)
+ if err:
+ return err
+ start, end = report_service.parse_range(request.args)
+ return jsonify(report_service.referrers(site_id, start, end)), 200
+
+
+@analytics_bp.route('/sites/<int:site_id>/devices', methods=['GET'])
+@viewer_required
+def site_devices(site_id):
+ site, err = _site_or_404(site_id)
+ if err:
+ return err
+ start, end = report_service.parse_range(request.args)
+ return jsonify(report_service.devices(site_id, start, end)), 200
+
+
+@analytics_bp.route('/sites/<int:site_id>/realtime', methods=['GET'])
+@viewer_required
+def site_realtime(site_id):
+ site, err = _site_or_404(site_id)
+ if err:
+ return err
+ minutes = request.args.get('minutes', 30)
+ return jsonify(report_service.realtime(site_id, minutes)), 200
+
+
+@analytics_bp.route('/rollup', methods=['POST'])
+@admin_required
+def trigger_rollup():
+ """Run the rollup on demand (ops/testing convenience)."""
+ return jsonify(rollup_service.run_rollup()), 200
diff --git a/builtin-extensions/serverkit-analytics/backend/report_service.py b/builtin-extensions/serverkit-analytics/backend/report_service.py
new file mode 100644
index 00000000..57b5e4f3
--- /dev/null
+++ b/builtin-extensions/serverkit-analytics/backend/report_service.py
@@ -0,0 +1,169 @@
+"""Read-side report queries for the serverkit-analytics dashboard.
+
+Everything the dashboard renders comes from the ``daily`` rollup table (fast,
+long-retention) EXCEPT the realtime tab, which reads the raw ``event`` table for
+the last N minutes.
+
+Range caveat (documented in docs/ANALYTICS.md): "unique visitors" over a
+multi-day range is the SUM of each day's uniques — a visitor active on two days
+counts twice. This is inherent to daily rollups and, combined with cookieless
+hashing, means visitor counts are close approximations, not exact identities.
+Pageviews sum exactly.
+"""
+from datetime import date, datetime, timedelta
+
+_RANGE_DAYS = {'1d': 1, '7d': 7, '14d': 14, '30d': 30, '90d': 90, '180d': 180,
+ '365d': 365, '12mo': 365, '13mo': 396}
+
+
+def parse_range(args):
+ """Resolve (start_date, end_date) from query args.
+
+ Accepts ``?range=7d`` (default 7d) or explicit ``?start=YYYY-MM-DD&
+ end=YYYY-MM-DD``. End defaults to today (UTC), inclusive.
+ """
+ today = datetime.utcnow().date()
+ start_raw = (args.get('start') or '').strip()
+ end_raw = (args.get('end') or '').strip()
+ if start_raw:
+ try:
+ start = datetime.strptime(start_raw[:10], '%Y-%m-%d').date()
+ end = (datetime.strptime(end_raw[:10], '%Y-%m-%d').date()
+ if end_raw else today)
+ if end < start:
+ start, end = end, start
+ return start, end
+ except ValueError:
+ pass
+ days = _RANGE_DAYS.get((args.get('range') or '7d').strip(), 7)
+ return today - timedelta(days=days - 1), today
+
+
+def _overall_query(site_id, start, end):
+ from .models import AnalyticsDaily
+ return (AnalyticsDaily.query
+ .filter(AnalyticsDaily.site_id == site_id,
+ AnalyticsDaily.dim_type == 'overall',
+ AnalyticsDaily.date >= start,
+ AnalyticsDaily.date <= end))
+
+
+def _totals(site_id, start, end):
+ from app import db
+ from sqlalchemy import func
+ from .models import AnalyticsDaily
+ row = (db.session.query(
+ func.coalesce(func.sum(AnalyticsDaily.visitors), 0),
+ func.coalesce(func.sum(AnalyticsDaily.pageviews), 0),
+ func.coalesce(func.sum(AnalyticsDaily.bounces), 0),
+ func.avg(AnalyticsDaily.avg_load_ms),
+ ).filter(AnalyticsDaily.site_id == site_id,
+ AnalyticsDaily.dim_type == 'overall',
+ AnalyticsDaily.date >= start,
+ AnalyticsDaily.date <= end).one())
+ visitors, pageviews, bounces, avg_load = row
+ bounce_rate = round(100.0 * bounces / visitors, 1) if visitors else 0.0
+ return {
+ 'visitors': int(visitors or 0),
+ 'pageviews': int(pageviews or 0),
+ 'bounces': int(bounces or 0),
+ 'bounce_rate': bounce_rate,
+ 'avg_load_ms': round(avg_load, 1) if avg_load is not None else None,
+ }
+
+
+def timeseries(site_id, start, end):
+ """Per-day visitors + pageviews across the range, zero-filled."""
+ rows = {r.date: r for r in _overall_query(site_id, start, end).all()}
+ out = []
+ d = start
+ while d <= end:
+ r = rows.get(d)
+ out.append({
+ 'date': d.isoformat(),
+ 'visitors': r.visitors if r else 0,
+ 'pageviews': r.pageviews if r else 0,
+ })
+ d += timedelta(days=1)
+ return out
+
+
+def _dim_table(site_id, start, end, dim_type, limit=None):
+ from app import db
+ from sqlalchemy import func
+ from .models import AnalyticsDaily
+ q = (db.session.query(
+ AnalyticsDaily.dim_value,
+ func.sum(AnalyticsDaily.visitors).label('visitors'),
+ func.sum(AnalyticsDaily.pageviews).label('pageviews'),
+ func.avg(AnalyticsDaily.avg_load_ms).label('avg_load'),
+ ).filter(AnalyticsDaily.site_id == site_id,
+ AnalyticsDaily.dim_type == dim_type,
+ AnalyticsDaily.date >= start,
+ AnalyticsDaily.date <= end)
+ .group_by(AnalyticsDaily.dim_value)
+ .order_by(func.sum(AnalyticsDaily.pageviews).desc()))
+ if limit:
+ q = q.limit(limit)
+ return [{
+ 'value': value or '',
+ 'visitors': int(visitors or 0),
+ 'pageviews': int(pageviews or 0),
+ 'avg_load_ms': round(avg_load, 1) if avg_load is not None else None,
+ } for value, visitors, pageviews, avg_load in q.all()]
+
+
+def realtime(site_id, minutes=30):
+ """Live view from RAW events (last N minutes): active visitors, pageviews,
+ and the most recent hits."""
+ from .models import AnalyticsEvent
+ minutes = max(1, min(int(minutes or 30), 1440))
+ since = datetime.utcnow() - timedelta(minutes=minutes)
+ q = (AnalyticsEvent.query
+ .filter(AnalyticsEvent.site_id == site_id, AnalyticsEvent.ts >= since))
+ events = q.order_by(AnalyticsEvent.ts.desc()).limit(100).all()
+ active = {e.visitor_hash for e in q.all() if e.visitor_hash}
+ return {
+ 'minutes': minutes,
+ 'active_visitors': len(active),
+ 'pageviews': q.filter(AnalyticsEvent.type == 'pageview').count(),
+ 'recent': [{
+ 'ts': e.ts.isoformat() if e.ts else None,
+ 'path': e.url_path,
+ 'referrer_host': e.referrer_host,
+ 'device_class': e.device_class,
+ 'country': e.country,
+ } for e in events],
+ }
+
+
+def overview(site_id, start, end):
+ """Headline KPIs + sparkline + top pages/referrers + a live counter."""
+ return {
+ 'range': {'start': start.isoformat(), 'end': end.isoformat()},
+ 'totals': _totals(site_id, start, end),
+ 'timeseries': timeseries(site_id, start, end),
+ 'top_pages': _dim_table(site_id, start, end, 'page', limit=8),
+ 'top_referrers': _dim_table(site_id, start, end, 'referrer', limit=8),
+ 'realtime': realtime(site_id, minutes=30),
+ }
+
+
+def pages(site_id, start, end, limit=200):
+ return {'range': {'start': start.isoformat(), 'end': end.isoformat()},
+ 'rows': _dim_table(site_id, start, end, 'page', limit=limit)}
+
+
+def referrers(site_id, start, end, limit=200):
+ return {'range': {'start': start.isoformat(), 'end': end.isoformat()},
+ 'rows': _dim_table(site_id, start, end, 'referrer', limit=limit)}
+
+
+def devices(site_id, start, end):
+ return {
+ 'range': {'start': start.isoformat(), 'end': end.isoformat()},
+ 'device': _dim_table(site_id, start, end, 'device'),
+ 'browser': _dim_table(site_id, start, end, 'browser'),
+ 'os': _dim_table(site_id, start, end, 'os'),
+ 'country': _dim_table(site_id, start, end, 'country'),
+ }
diff --git a/builtin-extensions/serverkit-analytics/backend/rollup_service.py b/builtin-extensions/serverkit-analytics/backend/rollup_service.py
index 67437575..743dbb6a 100644
--- a/builtin-extensions/serverkit-analytics/backend/rollup_service.py
+++ b/builtin-extensions/serverkit-analytics/backend/rollup_service.py
@@ -46,6 +46,106 @@ def run_retention_prune():
}
+def _as_date(value):
+ """Normalize a grouped day value (SQLite returns a str, PG a date)."""
+ if isinstance(value, str):
+ return datetime.strptime(value[:10], '%Y-%m-%d').date()
+ if isinstance(value, datetime):
+ return value.date()
+ return value
+
+
+# Dimension rollups: (dim_type, event column attribute, skip-null?).
+_DIMENSIONS = [
+ ('page', 'url_path', False),
+ ('referrer', 'referrer_host', True),
+ ('device', 'device_class', False),
+ ('browser', 'ua_family', False),
+ ('os', 'os_family', False),
+ ('country', 'country', True),
+]
+
+
def run_rollup():
- """Aggregate raw events into the daily rollup table. Filled in Phase 2."""
- return {'skipped': True, 'reason': 'rollup implemented in Phase 2'}
+ """Aggregate raw pageview events into the ``daily`` rollup table.
+
+ Idempotent and backfill-safe: recomputes every (site, date) that currently
+ has raw events (bounded by ``raw_retention_days``), deleting and re-inserting
+ that day's rollup rows. Aggregation runs DB-side (grouped queries) so only
+ summary rows — not raw events — reach Python; on a very busy SQLite panel the
+ per-hour recompute is the documented pressure point (recommend PostgreSQL).
+
+ Only ``type == 'pageview'`` events feed the rollup in v1; outlink/download/
+ custom events are stored but not summarized here.
+ """
+ from app import db
+ from sqlalchemy import distinct, func
+ from .models import AnalyticsDaily, AnalyticsEvent
+
+ day = func.date(AnalyticsEvent.ts)
+ pageview = AnalyticsEvent.type == 'pageview'
+
+ pairs = (db.session.query(AnalyticsEvent.site_id, day)
+ .filter(pageview).distinct().all())
+ date_set = {(sid, _as_date(d)) for sid, d in pairs}
+ if not date_set:
+ return {'sites_dates': 0, 'rows': 0}
+
+ # Clear the days we're about to recompute (keeps late arrivals correct).
+ for sid, d in date_set:
+ AnalyticsDaily.query.filter_by(site_id=sid, date=d).delete(
+ synchronize_session=False)
+
+ rows = 0
+
+ # --- overall (per site+day) + bounce (single-pageview visitors) ---
+ overall = (db.session.query(
+ AnalyticsEvent.site_id, day.label('d'),
+ func.count(distinct(AnalyticsEvent.visitor_hash)).label('visitors'),
+ func.count().label('pageviews'),
+ func.avg(AnalyticsEvent.load_ms).label('avg_load'),
+ ).filter(pageview).group_by(AnalyticsEvent.site_id, day).all())
+
+ per_visitor = (db.session.query(
+ AnalyticsEvent.site_id, day.label('d'), AnalyticsEvent.visitor_hash,
+ func.count().label('c'),
+ ).filter(pageview)
+ .group_by(AnalyticsEvent.site_id, day, AnalyticsEvent.visitor_hash)
+ .subquery())
+ bounce_rows = (db.session.query(
+ per_visitor.c.site_id, per_visitor.c.d, func.count().label('b'))
+ .filter(per_visitor.c.c == 1)
+ .group_by(per_visitor.c.site_id, per_visitor.c.d).all())
+ bounce_map = {(sid, _as_date(d)): b for sid, d, b in bounce_rows}
+
+ for sid, d, visitors, pageviews, avg_load in overall:
+ dd = _as_date(d)
+ db.session.add(AnalyticsDaily(
+ site_id=sid, date=dd, dim_type='overall', dim_value='',
+ visitors=visitors or 0, visits=visitors or 0,
+ pageviews=pageviews or 0, bounces=bounce_map.get((sid, dd), 0),
+ avg_load_ms=avg_load))
+ rows += 1
+
+ # --- dimension rollups ---
+ for dim_type, attr, skip_null in _DIMENSIONS:
+ col = getattr(AnalyticsEvent, attr)
+ q = db.session.query(
+ AnalyticsEvent.site_id, day.label('d'), col.label('v'),
+ func.count(distinct(AnalyticsEvent.visitor_hash)).label('visitors'),
+ func.count().label('pageviews'),
+ func.avg(AnalyticsEvent.load_ms).label('avg_load'),
+ ).filter(pageview)
+ if skip_null:
+ q = q.filter(col.isnot(None))
+ q = q.group_by(AnalyticsEvent.site_id, day, col)
+ for sid, d, value, visitors, pageviews, avg_load in q.all():
+ db.session.add(AnalyticsDaily(
+ site_id=sid, date=_as_date(d), dim_type=dim_type,
+ dim_value=(str(value) if value is not None else '')[:512],
+ visitors=visitors or 0, visits=visitors or 0,
+ pageviews=pageviews or 0, bounces=0, avg_load_ms=avg_load))
+ rows += 1
+
+ db.session.commit()
+ return {'sites_dates': len(date_set), 'rows': rows}
diff --git a/builtin-extensions/serverkit-analytics/backend/site_service.py b/builtin-extensions/serverkit-analytics/backend/site_service.py
new file mode 100644
index 00000000..f951e63f
--- /dev/null
+++ b/builtin-extensions/serverkit-analytics/backend/site_service.py
@@ -0,0 +1,105 @@
+"""Tracked-site CRUD for serverkit-analytics.
+
+A ``site`` groups one or more hostnames under a public ``site_key`` (baked into
+the tracker snippet). Mutations are admin-gated at the route layer; this module
+is pure data logic and raises ``ValueError`` for bad input (routes translate to
+400/404).
+"""
+from .models import AnalyticsSite, generate_site_key
+
+
+def _normalize_hostnames(value):
+ """Accept a list or a comma/newline string; return a newline-joined string."""
+ if value is None:
+ return None
+ if isinstance(value, (list, tuple)):
+ items = [str(v).strip() for v in value if str(v).strip()]
+ else:
+ items = [c.strip() for c in str(value).replace(',', '\n').splitlines()
+ if c.strip()]
+ return '\n'.join(items) if items else None
+
+
+def list_sites():
+ return [s.to_dict() for s in AnalyticsSite.query.order_by(AnalyticsSite.id).all()]
+
+
+def get_site(site_id):
+ return AnalyticsSite.query.get(site_id)
+
+
+def get_by_key(site_key):
+ return AnalyticsSite.query.filter_by(site_key=site_key).first()
+
+
+def create_site(data):
+ from app import db
+ name = (data.get('name') or '').strip()
+ if not name:
+ raise ValueError('name is required')
+ site = AnalyticsSite(
+ name=name[:255],
+ hostnames=_normalize_hostnames(data.get('hostnames')),
+ site_key=generate_site_key(),
+ created_from=(data.get('created_from') or 'manual')[:32],
+ app_id=data.get('app_id'),
+ honor_dnt=data.get('honor_dnt'),
+ allowed_origins=_normalize_hostnames(data.get('allowed_origins')),
+ enabled=bool(data.get('enabled', True)),
+ )
+ if data.get('settings'):
+ site.set_settings(data.get('settings'))
+ db.session.add(site)
+ db.session.commit()
+ return site
+
+
+def update_site(site_id, data):
+ from app import db
+ site = get_site(site_id)
+ if not site:
+ return None
+ if 'name' in data:
+ name = (data.get('name') or '').strip()
+ if not name:
+ raise ValueError('name cannot be blank')
+ site.name = name[:255]
+ if 'hostnames' in data:
+ site.hostnames = _normalize_hostnames(data.get('hostnames'))
+ if 'allowed_origins' in data:
+ site.allowed_origins = _normalize_hostnames(data.get('allowed_origins'))
+ if 'honor_dnt' in data:
+ site.honor_dnt = data.get('honor_dnt')
+ if 'enabled' in data:
+ site.enabled = bool(data.get('enabled'))
+ if 'app_id' in data:
+ site.app_id = data.get('app_id')
+ if 'settings' in data and isinstance(data.get('settings'), dict):
+ site.update_settings(**data['settings'])
+ db.session.commit()
+ return site
+
+
+def rotate_key(site_id):
+ from app import db
+ site = get_site(site_id)
+ if not site:
+ return None
+ site.site_key = generate_site_key()
+ db.session.commit()
+ return site
+
+
+def delete_site(site_id):
+ """Delete a site and all of its data (events, rollups, log cursors)."""
+ from app import db
+ from .models import AnalyticsEvent, AnalyticsDaily, AnalyticsLogCursor
+ site = get_site(site_id)
+ if not site:
+ return False
+ AnalyticsEvent.query.filter_by(site_id=site_id).delete(synchronize_session=False)
+ AnalyticsDaily.query.filter_by(site_id=site_id).delete(synchronize_session=False)
+ AnalyticsLogCursor.query.filter_by(site_id=site_id).delete(synchronize_session=False)
+ db.session.delete(site)
+ db.session.commit()
+ return True