Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

feat(analytics): log ingestion + optional geo (Phase 6)

Juan Denis7bc323b41b86
diff --git a/backend/tests/test_analytics_extension.py b/backend/tests/test_analytics_extension.py
index 1c95e62c..0d8af9be 100644
--- a/backend/tests/test_analytics_extension.py
+++ b/backend/tests/test_analytics_extension.py
@@ -32,7 +32,8 @@ def _load_ext():
mods = {}
for name in ('config', 'models', 'ingest_service', 'rollup_service',
'report_service', 'site_service', 'wp_integration',
- 'nginx_integration', 'lifecycle', 'jobs', 'analytics'):
+ 'nginx_integration', 'log_ingest_service', 'geo', 'lifecycle',
+ 'jobs', 'analytics'):
mods[name] = importlib.import_module(f'app.plugins.{SLUG}.{name}')
return mods
@@ -46,6 +47,8 @@ report_mod = _M['report_service']
site_mod = _M['site_service']
wp_mod = _M['wp_integration']
nginx_mod = _M['nginx_integration']
+log_mod = _M['log_ingest_service']
+geo_mod = _M['geo']
lifecycle_mod = _M['lifecycle']
jobs_mod = _M['jobs']
bp_mod = _M['analytics']
@@ -925,3 +928,143 @@ def test_inject_endpoints_admin_gated(app, analytics_client, monkeypatch):
r = analytics_client.post(f'/api/v1/analytics/sites/{site.id}/inject/wordpress',
headers=admin)
assert r.status_code == 200 and r.get_json().get('success')
+
+
+# --------------------------------------------------------------------------- #
+# Phase 6: log ingestion + geo
+# --------------------------------------------------------------------------- #
+_LOG_LINE = (
+ '203.0.113.9 - - [10/Oct/2024:13:55:36 +0000] "GET /blog/post HTTP/1.1" '
+ '200 5120 "https://google.com/search" '
+ '"Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 Chrome/120 Safari/537.36"'
+)
+_LOG_ASSET = (
+ '203.0.113.9 - - [10/Oct/2024:13:55:37 +0000] "GET /style.css HTTP/1.1" '
+ '200 1000 "-" "Mozilla/5.0 Chrome/120"'
+)
+_LOG_BOT = (
+ '203.0.113.9 - - [10/Oct/2024:13:55:38 +0000] "GET / HTTP/1.1" '
+ '200 1000 "-" "Googlebot/2.1"'
+)
+_LOG_POST = (
+ '203.0.113.9 - - [10/Oct/2024:13:55:39 +0000] "POST /wp-login.php HTTP/1.1" '
+ '200 1000 "-" "Mozilla/5.0 Chrome/120"'
+)
+
+
+def test_parse_combined_line():
+ p = log_mod.parse_combined_line(_LOG_LINE)
+ assert p['ip'] == '203.0.113.9'
+ assert p['path'] == '/blog/post'
+ assert p['status'] == '200'
+ assert 'google' in p['referer']
+ # assets, bots-by-request, and non-GET are filtered at parse time.
+ assert log_mod.parse_combined_line(_LOG_ASSET) is None # asset
+ assert log_mod.parse_combined_line(_LOG_POST) is None # non-GET
+ assert log_mod.parse_combined_line('garbage line') is None
+
+
+def test_ingest_lines_creates_log_events(app):
+ site = _mk_site()
+ written, max_ts = log_mod.ingest_lines(site, [_LOG_LINE, _LOG_ASSET, _LOG_BOT])
+ assert written == 1 # only the real pageview (asset + bot dropped)
+ ev = AnalyticsEvent.query.filter_by(site_id=site.id).one()
+ assert ev.source == 'log'
+ assert ev.url_path == '/blog/post'
+ assert ev.referrer_host == 'google.com'
+ assert ev.ua_family == 'Chrome'
+ assert ev.visitor_hash and '203.0.113.9' not in ev.visitor_hash
+
+
+def test_ingest_lines_dedup_by_since(app):
+ from datetime import datetime
+ site = _mk_site()
+ # since_ts at/after the line's ts => skipped.
+ boundary = datetime(2024, 10, 10, 14, 0, 0)
+ written, _ = log_mod.ingest_lines(site, [_LOG_LINE], since_ts=boundary)
+ assert written == 0
+
+
+def test_log_file_cursor_advances(app, tmp_path):
+ site = _mk_site()
+ logf = tmp_path / 'access.log'
+ logf.write_text(_LOG_LINE + '\n', encoding='utf-8')
+ site.update_settings(log_ingestion=True, log_source_kind='file',
+ log_source_ref=str(logf))
+ from app import db
+ db.session.commit()
+
+ res1 = log_mod.ingest_site(site)
+ assert res1['written'] == 1
+ # Second run with no new lines writes nothing (cursor advanced).
+ res2 = log_mod.ingest_site(site)
+ assert res2['written'] == 0
+ # Append a new line -> ingested on the next run.
+ with open(logf, 'a', encoding='utf-8') as f:
+ f.write(_LOG_LINE.replace('/blog/post', '/blog/post2') + '\n')
+ res3 = log_mod.ingest_site(site)
+ assert res3['written'] == 1
+ assert AnalyticsEvent.query.filter_by(site_id=site.id).count() == 2
+
+
+def test_ingest_site_skips_when_disabled(app):
+ site = _mk_site() # no log_ingestion setting
+ assert log_mod.ingest_site(site).get('skipped') is True
+
+
+def test_run_log_tail_iterates_enabled_sites(app, tmp_path):
+ _mk_plugin_row(config={'log_ingestion_enabled': True})
+ site = _mk_site()
+ logf = tmp_path / 'a.log'
+ logf.write_text(_LOG_LINE + '\n', encoding='utf-8')
+ site.update_settings(log_ingestion=True, log_source_kind='file',
+ log_source_ref=str(logf))
+ from app import db
+ db.session.commit()
+ out = log_mod.run_log_tail()
+ assert out['sites'] == 1 and out['written'] == 1
+
+
+def test_run_log_tail_respects_global_toggle(app):
+ _mk_plugin_row(config={'log_ingestion_enabled': False})
+ out = log_mod.run_log_tail()
+ assert out.get('skipped') is True
+
+
+def test_log_tail_job_handler_never_raises(app):
+ _mk_plugin_row(config={'log_ingestion_enabled': False})
+ out = jobs_mod.log_tail(job=None)
+ assert 'skipped' in out or 'sites' in out or 'error' in out
+
+
+def test_ingest_logs_endpoint(app, analytics_client, tmp_path):
+ admin = _auth(_mk_admin())
+ site = _mk_site()
+ logf = tmp_path / 'x.log'
+ logf.write_text(_LOG_LINE + '\n', encoding='utf-8')
+ site.update_settings(log_ingestion=True, log_source_kind='file',
+ log_source_ref=str(logf))
+ from app import db
+ db.session.commit()
+ r = analytics_client.post(f'/api/v1/analytics/sites/{site.id}/ingest-logs',
+ headers=admin)
+ assert r.status_code == 200 and r.get_json()['written'] == 1
+
+
+# --- geo ---
+def test_geo_disabled_returns_none(app):
+ _mk_plugin_row(config={'geo_enabled': False})
+ assert geo_mod.lookup_country('8.8.8.8') is None
+
+
+def test_geo_enabled_but_no_db(app):
+ _mk_plugin_row(config={'geo_enabled': True, 'geo_db_path': '/nonexistent/GeoLite2.mmdb'})
+ geo_mod.reset()
+ assert geo_mod.lookup_country('8.8.8.8') is None
+
+
+def test_build_event_country_none_by_default(app):
+ site = _mk_site()
+ event = ingest_mod.build_event(site, {'k': site.site_key, 'p': '/x'},
+ '8.8.8.8', UA_CHROME)
+ assert event['country'] is None
diff --git a/builtin-extensions/serverkit-analytics/backend/analytics.py b/builtin-extensions/serverkit-analytics/backend/analytics.py
index 54629940..e60e7c43 100644
--- a/builtin-extensions/serverkit-analytics/backend/analytics.py
+++ b/builtin-extensions/serverkit-analytics/backend/analytics.py
@@ -27,8 +27,8 @@ from app.middleware.rbac import admin_required, viewer_required
from app.utils.client_ip import get_client_ip
from . import (
- ingest_service, nginx_integration, report_service, rollup_service,
- site_service, wp_integration,
+ ingest_service, log_ingest_service, nginx_integration, report_service,
+ rollup_service, site_service, wp_integration,
)
from .config import cfg_bool
from .ingest_service import MAX_BODY_BYTES, referrer_host
@@ -408,3 +408,13 @@ def remove_nginx(site_id):
if err:
return err
return jsonify(nginx_integration.remove(site)), 200
+
+
+@analytics_bp.route('/sites/<int:site_id>/ingest-logs', methods=['POST'])
+@admin_required
+def ingest_logs(site_id):
+ """Run server-log ingestion for one site on demand (ops/testing)."""
+ site, err = _site_or_404(site_id)
+ if err:
+ return err
+ return jsonify(log_ingest_service.ingest_site(site)), 200
diff --git a/builtin-extensions/serverkit-analytics/backend/geo.py b/builtin-extensions/serverkit-analytics/backend/geo.py
new file mode 100644
index 00000000..8889aed6
--- /dev/null
+++ b/builtin-extensions/serverkit-analytics/backend/geo.py
@@ -0,0 +1,69 @@
+"""Optional country-level geolocation for serverkit-analytics.
+
+Off by default. When the operator enables ``geo_enabled`` AND drops a
+GeoLite2-Country MMDB at ``geo_db_path``, we fill a 2-letter country code from
+the client IP. No database ships with the panel (licensing), and no IP is ever
+stored — only the derived country. Any failure (feature off, no DB, no
+``geoip2`` package, private IP) yields ``None`` and the country column stays
+null.
+"""
+import logging
+
+logger = logging.getLogger(__name__)
+
+# Cache one reader per DB path so we don't reopen the MMDB on every hit.
+_readers = {}
+_unavailable = set()
+
+
+def _reader(path):
+ if path in _unavailable:
+ return None
+ reader = _readers.get(path)
+ if reader is not None:
+ return reader
+ try:
+ import geoip2.database # noqa: PLC0415 - optional dependency
+ reader = geoip2.database.Reader(path)
+ _readers[path] = reader
+ return reader
+ except Exception as e: # noqa: BLE001 - missing package / bad file
+ logger.debug('geoip2 reader unavailable for %s: %s', path, e)
+ _unavailable.add(path)
+ return None
+
+
+def lookup_country(ip):
+ """Return an ISO-3166-1 alpha-2 country code for ``ip``, or ``None``."""
+ if not ip:
+ return None
+ try:
+ from .config import cfg_bool, cfg_str
+ if not cfg_bool('geo_enabled'):
+ return None
+ path = cfg_str('geo_db_path')
+ if not path:
+ return None
+ import os
+ if not os.path.exists(path):
+ return None
+ reader = _reader(path)
+ if reader is None:
+ return None
+ resp = reader.country(ip)
+ code = getattr(getattr(resp, 'country', None), 'iso_code', None)
+ return code or None
+ except Exception as e: # noqa: BLE001 - private/unknown IP, lookup miss
+ logger.debug('geo lookup miss for %s: %s', ip, e)
+ return None
+
+
+def reset():
+ """Test/shutdown helper — drop cached readers."""
+ for reader in _readers.values():
+ try:
+ reader.close()
+ except Exception: # noqa: BLE001
+ pass
+ _readers.clear()
+ _unavailable.clear()
diff --git a/builtin-extensions/serverkit-analytics/backend/ingest_service.py b/builtin-extensions/serverkit-analytics/backend/ingest_service.py
index 79f6ffa4..e1b70c2d 100644
--- a/builtin-extensions/serverkit-analytics/backend/ingest_service.py
+++ b/builtin-extensions/serverkit-analytics/backend/ingest_service.py
@@ -352,7 +352,15 @@ def build_event(site, payload, ip, ua):
'device_class': device,
'screen_class': screen,
'lang': lang,
- 'country': None, # filled by geo in Phase 6 when a GeoLite2 DB is present
+ 'country': _country(ip), # None unless geo is enabled + a GeoLite2 DB is present
'source': 'js',
'load_ms': load_ms,
}
+
+
+def _country(ip):
+ try:
+ from .geo import lookup_country
+ return lookup_country(ip)
+ except Exception: # noqa: BLE001
+ return None
diff --git a/builtin-extensions/serverkit-analytics/backend/log_ingest_service.py b/builtin-extensions/serverkit-analytics/backend/log_ingest_service.py
new file mode 100644
index 00000000..e6ad9670
--- /dev/null
+++ b/builtin-extensions/serverkit-analytics/backend/log_ingest_service.py
@@ -0,0 +1,252 @@
+"""Server-log ingestion for serverkit-analytics.
+
+A complementary, script-free source: parse apache/nginx **combined** access logs
+into the same event table as the JS tracker, tagged ``source='log'`` so JS-source
+and log-source hits stay comparable. Generalizes the on-demand
+``WpAnalyticsService.get_traffic`` parse into a persistent, incremental one.
+
+Per-site opt-in via ``site.settings``:
+ log_ingestion : bool (enable)
+ log_source_kind : 'file' | 'docker'
+ log_source_ref : file path (file) or container name (docker)
+
+Incrementality uses :class:`AnalyticsLogCursor`:
+* file — (inode, byte_offset); a changed inode or shrunken file resets to 0
+ (rotation/truncation safe).
+* docker — ``docker logs --since <last_ts>`` with a ts-based skip, since docker
+ logs has no byte offset.
+
+Only page-like GET hits are counted (assets and non-GET are skipped) and bots
+are dropped, mirroring the JS collector so the two sources line up. Visitor
+hashing reuses the collector's daily salt; log lines older than today are hashed
+with today's salt (a documented cookieless imprecision — tail runs every few
+minutes, so this is the rare backfill case).
+
+Linux-only at runtime (docker logs / access-log files); reading seams are
+mockable for tests.
+"""
+import logging
+import os
+import re
+import subprocess
+from datetime import datetime
+
+from .config import cfg_bool
+from .ingest_service import is_bot, parse_ua, referrer_host, visitor_hash
+
+logger = logging.getLogger(__name__)
+
+# apache/nginx "combined" log format:
+# %h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-Agent}i"
+COMBINED_RE = re.compile(
+ r'(?P<ip>\S+) \S+ \S+ \[(?P<time>[^\]]+)\] "(?P<request>[^"]*)" '
+ r'(?P<status>\d{3}) (?P<bytes>\S+) "(?P<referer>[^"]*)" "(?P<ua>[^"]*)"'
+)
+_TIME_FMT = '%d/%b/%Y:%H:%M:%S %z'
+
+# Skip static assets so log hits approximate real pageviews.
+_ASSET_RE = re.compile(
+ r'\.(?:css|js|mjs|png|jpe?g|gif|svg|webp|ico|bmp|woff2?|ttf|eot|otf|map|'
+ r'json|xml|txt|mp4|webm|mp3|wav|pdf|zip|gz)(?:$|\?)', re.IGNORECASE)
+
+# Cap lines processed per source per run so a huge backlog can't stall a job.
+MAX_LINES_PER_RUN = 20000
+
+
+def parse_combined_line(line):
+ """Parse one combined-log line into a dict, or ``None`` if it doesn't match
+ or isn't a page-like GET."""
+ m = COMBINED_RE.match(line.strip())
+ if not m:
+ return None
+ request = m.group('request') or ''
+ parts = request.split()
+ if len(parts) < 2 or parts[0].upper() != 'GET':
+ return None
+ path = parts[1].split('#', 1)[0]
+ if _ASSET_RE.search(path):
+ return None
+ try:
+ ts = datetime.strptime(m.group('time'), _TIME_FMT)
+ ts = ts.replace(tzinfo=None) - _utc_offset(ts)
+ except (ValueError, TypeError):
+ ts = datetime.utcnow()
+ referer = m.group('referer')
+ return {
+ 'ip': m.group('ip'),
+ 'ts': ts,
+ 'path': path.split('?', 1)[0][:512],
+ 'status': m.group('status'),
+ 'referer': None if referer in ('', '-') else referer,
+ 'ua': m.group('ua') or '',
+ }
+
+
+def _utc_offset(dt):
+ from datetime import timedelta
+ off = dt.utcoffset()
+ return off if off is not None else timedelta(0)
+
+
+def build_log_event(site, parsed):
+ """Build a persisted-event dict from a parsed log line (source='log')."""
+ ua = parsed['ua']
+ if not ua or is_bot(ua):
+ return None
+ browser, os_family, device = parse_ua(ua)
+ country = None
+ try:
+ from .geo import lookup_country
+ country = lookup_country(parsed['ip'])
+ except Exception: # noqa: BLE001
+ country = None
+ return {
+ 'site_id': site.id,
+ 'ts': parsed['ts'],
+ 'type': 'pageview',
+ 'event_name': None,
+ 'url_path': parsed['path'],
+ 'url_query': None,
+ 'referrer_host': referrer_host(parsed['referer']),
+ 'visitor_hash': visitor_hash(site.id, parsed['ip'], ua),
+ 'ua_family': browser,
+ 'os_family': os_family,
+ 'device_class': device,
+ 'screen_class': None,
+ 'lang': None,
+ 'country': country,
+ 'source': 'log',
+ 'load_ms': None,
+ }
+
+
+def ingest_lines(site, lines, since_ts=None):
+ """Parse + persist log lines for a site. Returns (written, max_ts)."""
+ from app import db
+ from .models import AnalyticsEvent
+ written = 0
+ max_ts = since_ts
+ for line in lines[:MAX_LINES_PER_RUN]:
+ parsed = parse_combined_line(line)
+ if not parsed:
+ continue
+ if since_ts and parsed['ts'] <= since_ts:
+ continue # already ingested (docker --since boundary)
+ event = build_log_event(site, parsed)
+ if not event:
+ continue
+ db.session.add(AnalyticsEvent(**event))
+ written += 1
+ if max_ts is None or parsed['ts'] > max_ts:
+ max_ts = parsed['ts']
+ if written:
+ db.session.commit()
+ return written, max_ts
+
+
+# --------------------------------------------------------------------------- #
+# source readers (mockable)
+# --------------------------------------------------------------------------- #
+def _read_file_lines(cursor, path):
+ """Return new lines from a file, advancing the (inode, byte_offset) cursor."""
+ try:
+ st = os.stat(path)
+ except OSError:
+ return []
+ inode = str(getattr(st, 'st_ino', '') or '')
+ offset = cursor.byte_offset or 0
+ if cursor.inode and inode and cursor.inode != inode:
+ offset = 0 # rotated
+ if offset > st.st_size:
+ offset = 0 # truncated
+ try:
+ with open(path, 'rb') as f:
+ f.seek(offset)
+ data = f.read()
+ new_offset = f.tell()
+ except OSError:
+ return []
+ cursor.inode = inode
+ cursor.byte_offset = new_offset
+ return data.decode('utf-8', 'ignore').splitlines()
+
+
+def _read_docker_lines(cursor, container):
+ """Return recent access-log lines from a container's stdout."""
+ args = ['docker', 'logs']
+ if cursor.last_ts:
+ args += ['--since', cursor.last_ts.strftime('%Y-%m-%dT%H:%M:%S')]
+ else:
+ args += ['--tail', '2000']
+ args.append(container)
+ try:
+ result = subprocess.run(args, capture_output=True, text=True, timeout=25)
+ except Exception as e: # noqa: BLE001
+ logger.debug('docker logs failed for %s: %s', container, e)
+ return []
+ return (result.stdout or '').splitlines()
+
+
+# --------------------------------------------------------------------------- #
+# orchestration
+# --------------------------------------------------------------------------- #
+def _get_cursor(site_id, source_ref, source_kind):
+ from app import db
+ from .models import AnalyticsLogCursor
+ cur = AnalyticsLogCursor.query.filter_by(
+ site_id=site_id, source_ref=source_ref).first()
+ if not cur:
+ cur = AnalyticsLogCursor(site_id=site_id, source_ref=source_ref,
+ source_kind=source_kind, byte_offset=0)
+ db.session.add(cur)
+ db.session.commit()
+ return cur
+
+
+def ingest_site(site):
+ """Ingest new log lines for one site. Returns a status dict."""
+ settings = site.get_settings()
+ if not settings.get('log_ingestion'):
+ return {'skipped': True, 'reason': 'log ingestion off for site'}
+ kind = settings.get('log_source_kind') or 'docker'
+ ref = settings.get('log_source_ref')
+ if not ref:
+ return {'skipped': True, 'reason': 'no log source configured'}
+
+ cursor = _get_cursor(site.id, ref, kind)
+ if kind == 'file':
+ # File dedup is by byte offset, so DON'T also filter by ts (that would
+ # drop same-second appends that the offset already advanced past).
+ lines = _read_file_lines(cursor, ref)
+ since = None
+ else:
+ # docker logs --since re-reads the boundary second; ts-filter dedups it.
+ lines = _read_docker_lines(cursor, ref)
+ since = cursor.last_ts
+
+ written, max_ts = ingest_lines(site, lines, since_ts=since)
+ if max_ts:
+ cursor.last_ts = max_ts
+ from app import db
+ cursor.updated_at = datetime.utcnow()
+ db.session.commit()
+ return {'site_id': site.id, 'written': written, 'lines': len(lines)}
+
+
+def run_log_tail():
+ """Ingest logs for every opted-in site (called by the scheduled job)."""
+ if not cfg_bool('log_ingestion_enabled'):
+ return {'skipped': True, 'reason': 'log ingestion globally disabled'}
+ from .models import AnalyticsSite
+ total = 0
+ sites = 0
+ for site in AnalyticsSite.query.filter_by(enabled=True).all():
+ if not site.get_settings().get('log_ingestion'):
+ continue
+ try:
+ res = ingest_site(site)
+ total += res.get('written', 0)
+ sites += 1
+ except Exception as e: # noqa: BLE001
+ logger.warning('log ingest failed for site %s: %s', site.id, e)
+ return {'sites': sites, 'written': total}