ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
feat(analytics): log ingestion + optional geo (Phase 6)
Juan Denis7bc323b41b86
diff --git a/backend/tests/test_analytics_extension.py b/backend/tests/test_analytics_extension.pyindex 1c95e62c..0d8af9be 100644--- a/backend/tests/test_analytics_extension.py+++ b/backend/tests/test_analytics_extension.py@@ -32,7 +32,8 @@ def _load_ext():mods = {}for name in ('config', 'models', 'ingest_service', 'rollup_service','report_service', 'site_service', 'wp_integration',- 'nginx_integration', 'lifecycle', 'jobs', 'analytics'):+ 'nginx_integration', 'log_ingest_service', 'geo', 'lifecycle',+ 'jobs', 'analytics'):mods[name] = importlib.import_module(f'app.plugins.{SLUG}.{name}')return mods@@ -46,6 +47,8 @@ report_mod = _M['report_service']site_mod = _M['site_service']wp_mod = _M['wp_integration']nginx_mod = _M['nginx_integration']+log_mod = _M['log_ingest_service']+geo_mod = _M['geo']lifecycle_mod = _M['lifecycle']jobs_mod = _M['jobs']bp_mod = _M['analytics']@@ -925,3 +928,143 @@ def test_inject_endpoints_admin_gated(app, analytics_client, monkeypatch):r = analytics_client.post(f'/api/v1/analytics/sites/{site.id}/inject/wordpress',headers=admin)assert r.status_code == 200 and r.get_json().get('success')+++# --------------------------------------------------------------------------- #+# Phase 6: log ingestion + geo+# --------------------------------------------------------------------------- #+_LOG_LINE = (+ '203.0.113.9 - - [10/Oct/2024:13:55:36 +0000] "GET /blog/post HTTP/1.1" '+ '200 5120 "https://google.com/search" '+ '"Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 Chrome/120 Safari/537.36"'+)+_LOG_ASSET = (+ '203.0.113.9 - - [10/Oct/2024:13:55:37 +0000] "GET /style.css HTTP/1.1" '+ '200 1000 "-" "Mozilla/5.0 Chrome/120"'+)+_LOG_BOT = (+ '203.0.113.9 - - [10/Oct/2024:13:55:38 +0000] "GET / HTTP/1.1" '+ '200 1000 "-" "Googlebot/2.1"'+)+_LOG_POST = (+ '203.0.113.9 - - [10/Oct/2024:13:55:39 +0000] "POST /wp-login.php HTTP/1.1" '+ '200 1000 "-" "Mozilla/5.0 Chrome/120"'+)+++def test_parse_combined_line():+ p = log_mod.parse_combined_line(_LOG_LINE)+ assert p['ip'] == '203.0.113.9'+ assert p['path'] == '/blog/post'+ assert p['status'] == '200'+ assert 'google' in p['referer']+ # assets, bots-by-request, and non-GET are filtered at parse time.+ assert log_mod.parse_combined_line(_LOG_ASSET) is None # asset+ assert log_mod.parse_combined_line(_LOG_POST) is None # non-GET+ assert log_mod.parse_combined_line('garbage line') is None+++def test_ingest_lines_creates_log_events(app):+ site = _mk_site()+ written, max_ts = log_mod.ingest_lines(site, [_LOG_LINE, _LOG_ASSET, _LOG_BOT])+ assert written == 1 # only the real pageview (asset + bot dropped)+ ev = AnalyticsEvent.query.filter_by(site_id=site.id).one()+ assert ev.source == 'log'+ assert ev.url_path == '/blog/post'+ assert ev.referrer_host == 'google.com'+ assert ev.ua_family == 'Chrome'+ assert ev.visitor_hash and '203.0.113.9' not in ev.visitor_hash+++def test_ingest_lines_dedup_by_since(app):+ from datetime import datetime+ site = _mk_site()+ # since_ts at/after the line's ts => skipped.+ boundary = datetime(2024, 10, 10, 14, 0, 0)+ written, _ = log_mod.ingest_lines(site, [_LOG_LINE], since_ts=boundary)+ assert written == 0+++def test_log_file_cursor_advances(app, tmp_path):+ site = _mk_site()+ logf = tmp_path / 'access.log'+ logf.write_text(_LOG_LINE + '\n', encoding='utf-8')+ site.update_settings(log_ingestion=True, log_source_kind='file',+ log_source_ref=str(logf))+ from app import db+ db.session.commit()++ res1 = log_mod.ingest_site(site)+ assert res1['written'] == 1+ # Second run with no new lines writes nothing (cursor advanced).+ res2 = log_mod.ingest_site(site)+ assert res2['written'] == 0+ # Append a new line -> ingested on the next run.+ with open(logf, 'a', encoding='utf-8') as f:+ f.write(_LOG_LINE.replace('/blog/post', '/blog/post2') + '\n')+ res3 = log_mod.ingest_site(site)+ assert res3['written'] == 1+ assert AnalyticsEvent.query.filter_by(site_id=site.id).count() == 2+++def test_ingest_site_skips_when_disabled(app):+ site = _mk_site() # no log_ingestion setting+ assert log_mod.ingest_site(site).get('skipped') is True+++def test_run_log_tail_iterates_enabled_sites(app, tmp_path):+ _mk_plugin_row(config={'log_ingestion_enabled': True})+ site = _mk_site()+ logf = tmp_path / 'a.log'+ logf.write_text(_LOG_LINE + '\n', encoding='utf-8')+ site.update_settings(log_ingestion=True, log_source_kind='file',+ log_source_ref=str(logf))+ from app import db+ db.session.commit()+ out = log_mod.run_log_tail()+ assert out['sites'] == 1 and out['written'] == 1+++def test_run_log_tail_respects_global_toggle(app):+ _mk_plugin_row(config={'log_ingestion_enabled': False})+ out = log_mod.run_log_tail()+ assert out.get('skipped') is True+++def test_log_tail_job_handler_never_raises(app):+ _mk_plugin_row(config={'log_ingestion_enabled': False})+ out = jobs_mod.log_tail(job=None)+ assert 'skipped' in out or 'sites' in out or 'error' in out+++def test_ingest_logs_endpoint(app, analytics_client, tmp_path):+ admin = _auth(_mk_admin())+ site = _mk_site()+ logf = tmp_path / 'x.log'+ logf.write_text(_LOG_LINE + '\n', encoding='utf-8')+ site.update_settings(log_ingestion=True, log_source_kind='file',+ log_source_ref=str(logf))+ from app import db+ db.session.commit()+ r = analytics_client.post(f'/api/v1/analytics/sites/{site.id}/ingest-logs',+ headers=admin)+ assert r.status_code == 200 and r.get_json()['written'] == 1+++# --- geo ---+def test_geo_disabled_returns_none(app):+ _mk_plugin_row(config={'geo_enabled': False})+ assert geo_mod.lookup_country('8.8.8.8') is None+++def test_geo_enabled_but_no_db(app):+ _mk_plugin_row(config={'geo_enabled': True, 'geo_db_path': '/nonexistent/GeoLite2.mmdb'})+ geo_mod.reset()+ assert geo_mod.lookup_country('8.8.8.8') is None+++def test_build_event_country_none_by_default(app):+ site = _mk_site()+ event = ingest_mod.build_event(site, {'k': site.site_key, 'p': '/x'},+ '8.8.8.8', UA_CHROME)+ assert event['country'] is Nonediff --git a/builtin-extensions/serverkit-analytics/backend/analytics.py b/builtin-extensions/serverkit-analytics/backend/analytics.pyindex 54629940..e60e7c43 100644--- a/builtin-extensions/serverkit-analytics/backend/analytics.py+++ b/builtin-extensions/serverkit-analytics/backend/analytics.py@@ -27,8 +27,8 @@ from app.middleware.rbac import admin_required, viewer_requiredfrom app.utils.client_ip import get_client_ipfrom . import (- ingest_service, nginx_integration, report_service, rollup_service,- site_service, wp_integration,+ ingest_service, log_ingest_service, nginx_integration, report_service,+ rollup_service, site_service, wp_integration,)from .config import cfg_boolfrom .ingest_service import MAX_BODY_BYTES, referrer_host@@ -408,3 +408,13 @@ def remove_nginx(site_id):if err:return errreturn jsonify(nginx_integration.remove(site)), 200+++@analytics_bp.route('/sites/<int:site_id>/ingest-logs', methods=['POST'])+@admin_required+def ingest_logs(site_id):+ """Run server-log ingestion for one site on demand (ops/testing)."""+ site, err = _site_or_404(site_id)+ if err:+ return err+ return jsonify(log_ingest_service.ingest_site(site)), 200diff --git a/builtin-extensions/serverkit-analytics/backend/geo.py b/builtin-extensions/serverkit-analytics/backend/geo.pynew file mode 100644index 00000000..8889aed6--- /dev/null+++ b/builtin-extensions/serverkit-analytics/backend/geo.py@@ -0,0 +1,69 @@+"""Optional country-level geolocation for serverkit-analytics.++Off by default. When the operator enables ``geo_enabled`` AND drops a+GeoLite2-Country MMDB at ``geo_db_path``, we fill a 2-letter country code from+the client IP. No database ships with the panel (licensing), and no IP is ever+stored — only the derived country. Any failure (feature off, no DB, no+``geoip2`` package, private IP) yields ``None`` and the country column stays+null.+"""+import logging++logger = logging.getLogger(__name__)++# Cache one reader per DB path so we don't reopen the MMDB on every hit.+_readers = {}+_unavailable = set()+++def _reader(path):+ if path in _unavailable:+ return None+ reader = _readers.get(path)+ if reader is not None:+ return reader+ try:+ import geoip2.database # noqa: PLC0415 - optional dependency+ reader = geoip2.database.Reader(path)+ _readers[path] = reader+ return reader+ except Exception as e: # noqa: BLE001 - missing package / bad file+ logger.debug('geoip2 reader unavailable for %s: %s', path, e)+ _unavailable.add(path)+ return None+++def lookup_country(ip):+ """Return an ISO-3166-1 alpha-2 country code for ``ip``, or ``None``."""+ if not ip:+ return None+ try:+ from .config import cfg_bool, cfg_str+ if not cfg_bool('geo_enabled'):+ return None+ path = cfg_str('geo_db_path')+ if not path:+ return None+ import os+ if not os.path.exists(path):+ return None+ reader = _reader(path)+ if reader is None:+ return None+ resp = reader.country(ip)+ code = getattr(getattr(resp, 'country', None), 'iso_code', None)+ return code or None+ except Exception as e: # noqa: BLE001 - private/unknown IP, lookup miss+ logger.debug('geo lookup miss for %s: %s', ip, e)+ return None+++def reset():+ """Test/shutdown helper — drop cached readers."""+ for reader in _readers.values():+ try:+ reader.close()+ except Exception: # noqa: BLE001+ pass+ _readers.clear()+ _unavailable.clear()diff --git a/builtin-extensions/serverkit-analytics/backend/ingest_service.py b/builtin-extensions/serverkit-analytics/backend/ingest_service.pyindex 79f6ffa4..e1b70c2d 100644--- a/builtin-extensions/serverkit-analytics/backend/ingest_service.py+++ b/builtin-extensions/serverkit-analytics/backend/ingest_service.py@@ -352,7 +352,15 @@ def build_event(site, payload, ip, ua):'device_class': device,'screen_class': screen,'lang': lang,- 'country': None, # filled by geo in Phase 6 when a GeoLite2 DB is present+ 'country': _country(ip), # None unless geo is enabled + a GeoLite2 DB is present'source': 'js','load_ms': load_ms,}+++def _country(ip):+ try:+ from .geo import lookup_country+ return lookup_country(ip)+ except Exception: # noqa: BLE001+ return Nonediff --git a/builtin-extensions/serverkit-analytics/backend/log_ingest_service.py b/builtin-extensions/serverkit-analytics/backend/log_ingest_service.pynew file mode 100644index 00000000..e6ad9670--- /dev/null+++ b/builtin-extensions/serverkit-analytics/backend/log_ingest_service.py@@ -0,0 +1,252 @@+"""Server-log ingestion for serverkit-analytics.++A complementary, script-free source: parse apache/nginx **combined** access logs+into the same event table as the JS tracker, tagged ``source='log'`` so JS-source+and log-source hits stay comparable. Generalizes the on-demand+``WpAnalyticsService.get_traffic`` parse into a persistent, incremental one.++Per-site opt-in via ``site.settings``:+ log_ingestion : bool (enable)+ log_source_kind : 'file' | 'docker'+ log_source_ref : file path (file) or container name (docker)++Incrementality uses :class:`AnalyticsLogCursor`:+* file — (inode, byte_offset); a changed inode or shrunken file resets to 0+ (rotation/truncation safe).+* docker — ``docker logs --since <last_ts>`` with a ts-based skip, since docker+ logs has no byte offset.++Only page-like GET hits are counted (assets and non-GET are skipped) and bots+are dropped, mirroring the JS collector so the two sources line up. Visitor+hashing reuses the collector's daily salt; log lines older than today are hashed+with today's salt (a documented cookieless imprecision — tail runs every few+minutes, so this is the rare backfill case).++Linux-only at runtime (docker logs / access-log files); reading seams are+mockable for tests.+"""+import logging+import os+import re+import subprocess+from datetime import datetime++from .config import cfg_bool+from .ingest_service import is_bot, parse_ua, referrer_host, visitor_hash++logger = logging.getLogger(__name__)++# apache/nginx "combined" log format:+# %h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-Agent}i"+COMBINED_RE = re.compile(+ r'(?P<ip>\S+) \S+ \S+ \[(?P<time>[^\]]+)\] "(?P<request>[^"]*)" '+ r'(?P<status>\d{3}) (?P<bytes>\S+) "(?P<referer>[^"]*)" "(?P<ua>[^"]*)"'+)+_TIME_FMT = '%d/%b/%Y:%H:%M:%S %z'++# Skip static assets so log hits approximate real pageviews.+_ASSET_RE = re.compile(+ r'\.(?:css|js|mjs|png|jpe?g|gif|svg|webp|ico|bmp|woff2?|ttf|eot|otf|map|'+ r'json|xml|txt|mp4|webm|mp3|wav|pdf|zip|gz)(?:$|\?)', re.IGNORECASE)++# Cap lines processed per source per run so a huge backlog can't stall a job.+MAX_LINES_PER_RUN = 20000+++def parse_combined_line(line):+ """Parse one combined-log line into a dict, or ``None`` if it doesn't match+ or isn't a page-like GET."""+ m = COMBINED_RE.match(line.strip())+ if not m:+ return None+ request = m.group('request') or ''+ parts = request.split()+ if len(parts) < 2 or parts[0].upper() != 'GET':+ return None+ path = parts[1].split('#', 1)[0]+ if _ASSET_RE.search(path):+ return None+ try:+ ts = datetime.strptime(m.group('time'), _TIME_FMT)+ ts = ts.replace(tzinfo=None) - _utc_offset(ts)+ except (ValueError, TypeError):+ ts = datetime.utcnow()+ referer = m.group('referer')+ return {+ 'ip': m.group('ip'),+ 'ts': ts,+ 'path': path.split('?', 1)[0][:512],+ 'status': m.group('status'),+ 'referer': None if referer in ('', '-') else referer,+ 'ua': m.group('ua') or '',+ }+++def _utc_offset(dt):+ from datetime import timedelta+ off = dt.utcoffset()+ return off if off is not None else timedelta(0)+++def build_log_event(site, parsed):+ """Build a persisted-event dict from a parsed log line (source='log')."""+ ua = parsed['ua']+ if not ua or is_bot(ua):+ return None+ browser, os_family, device = parse_ua(ua)+ country = None+ try:+ from .geo import lookup_country+ country = lookup_country(parsed['ip'])+ except Exception: # noqa: BLE001+ country = None+ return {+ 'site_id': site.id,+ 'ts': parsed['ts'],+ 'type': 'pageview',+ 'event_name': None,+ 'url_path': parsed['path'],+ 'url_query': None,+ 'referrer_host': referrer_host(parsed['referer']),+ 'visitor_hash': visitor_hash(site.id, parsed['ip'], ua),+ 'ua_family': browser,+ 'os_family': os_family,+ 'device_class': device,+ 'screen_class': None,+ 'lang': None,+ 'country': country,+ 'source': 'log',+ 'load_ms': None,+ }+++def ingest_lines(site, lines, since_ts=None):+ """Parse + persist log lines for a site. Returns (written, max_ts)."""+ from app import db+ from .models import AnalyticsEvent+ written = 0+ max_ts = since_ts+ for line in lines[:MAX_LINES_PER_RUN]:+ parsed = parse_combined_line(line)+ if not parsed:+ continue+ if since_ts and parsed['ts'] <= since_ts:+ continue # already ingested (docker --since boundary)+ event = build_log_event(site, parsed)+ if not event:+ continue+ db.session.add(AnalyticsEvent(**event))+ written += 1+ if max_ts is None or parsed['ts'] > max_ts:+ max_ts = parsed['ts']+ if written:+ db.session.commit()+ return written, max_ts+++# --------------------------------------------------------------------------- #+# source readers (mockable)+# --------------------------------------------------------------------------- #+def _read_file_lines(cursor, path):+ """Return new lines from a file, advancing the (inode, byte_offset) cursor."""+ try:+ st = os.stat(path)+ except OSError:+ return []+ inode = str(getattr(st, 'st_ino', '') or '')+ offset = cursor.byte_offset or 0+ if cursor.inode and inode and cursor.inode != inode:+ offset = 0 # rotated+ if offset > st.st_size:+ offset = 0 # truncated+ try:+ with open(path, 'rb') as f:+ f.seek(offset)+ data = f.read()+ new_offset = f.tell()+ except OSError:+ return []+ cursor.inode = inode+ cursor.byte_offset = new_offset+ return data.decode('utf-8', 'ignore').splitlines()+++def _read_docker_lines(cursor, container):+ """Return recent access-log lines from a container's stdout."""+ args = ['docker', 'logs']+ if cursor.last_ts:+ args += ['--since', cursor.last_ts.strftime('%Y-%m-%dT%H:%M:%S')]+ else:+ args += ['--tail', '2000']+ args.append(container)+ try:+ result = subprocess.run(args, capture_output=True, text=True, timeout=25)+ except Exception as e: # noqa: BLE001+ logger.debug('docker logs failed for %s: %s', container, e)+ return []+ return (result.stdout or '').splitlines()+++# --------------------------------------------------------------------------- #+# orchestration+# --------------------------------------------------------------------------- #+def _get_cursor(site_id, source_ref, source_kind):+ from app import db+ from .models import AnalyticsLogCursor+ cur = AnalyticsLogCursor.query.filter_by(+ site_id=site_id, source_ref=source_ref).first()+ if not cur:+ cur = AnalyticsLogCursor(site_id=site_id, source_ref=source_ref,+ source_kind=source_kind, byte_offset=0)+ db.session.add(cur)+ db.session.commit()+ return cur+++def ingest_site(site):+ """Ingest new log lines for one site. Returns a status dict."""+ settings = site.get_settings()+ if not settings.get('log_ingestion'):+ return {'skipped': True, 'reason': 'log ingestion off for site'}+ kind = settings.get('log_source_kind') or 'docker'+ ref = settings.get('log_source_ref')+ if not ref:+ return {'skipped': True, 'reason': 'no log source configured'}++ cursor = _get_cursor(site.id, ref, kind)+ if kind == 'file':+ # File dedup is by byte offset, so DON'T also filter by ts (that would+ # drop same-second appends that the offset already advanced past).+ lines = _read_file_lines(cursor, ref)+ since = None+ else:+ # docker logs --since re-reads the boundary second; ts-filter dedups it.+ lines = _read_docker_lines(cursor, ref)+ since = cursor.last_ts++ written, max_ts = ingest_lines(site, lines, since_ts=since)+ if max_ts:+ cursor.last_ts = max_ts+ from app import db+ cursor.updated_at = datetime.utcnow()+ db.session.commit()+ return {'site_id': site.id, 'written': written, 'lines': len(lines)}+++def run_log_tail():+ """Ingest logs for every opted-in site (called by the scheduled job)."""+ if not cfg_bool('log_ingestion_enabled'):+ return {'skipped': True, 'reason': 'log ingestion globally disabled'}+ from .models import AnalyticsSite+ total = 0+ sites = 0+ for site in AnalyticsSite.query.filter_by(enabled=True).all():+ if not site.get_settings().get('log_ingestion'):+ continue+ try:+ res = ingest_site(site)+ total += res.get('written', 0)+ sites += 1+ except Exception as e: # noqa: BLE001+ logger.warning('log ingest failed for site %s: %s', site.id, e)+ return {'sites': sites, 'written': total}