Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

fix(connect): enforce consent and recover bounded log delivery [skip version]

Juan Denisb181e3267487
diff --git a/backend/app/services/connect_client.py b/backend/app/services/connect_client.py
index f6316957..c27bb784 100644
--- a/backend/app/services/connect_client.py
+++ b/backend/app/services/connect_client.py
@@ -702,12 +702,14 @@ class RelayClient:
self._diagnostics_stream = 0
self._diagnostics_next_at = 0.0
self._diagnostics_inflight = {}
+ self._diagnostics_collect = False
# Bounded log collection. The publisher owns the
# buffer and the bounds; Cloud's answer to each batch owns whether
# to keep going and which sources are wanted.
self._logs = connect_logs.LogPublisher(app)
self._logs_stream = 0
self._logs_inflight = {}
+ self._logs_ack_deadline = 0.0
# -- lifecycle -----------------------------------------------------
@@ -827,6 +829,12 @@ class RelayClient:
self._ws = ws
try:
+ # Consent is connection-scoped. Reconnect using empty probes.
+ if self._diagnostics_collect:
+ self._diagnostics_next_at = 0.0
+ self._diagnostics_collect = False
+ self._logs.sources = None
+ self._logs.next_flush_at = 0.0
self.transport = 'ws'
self._set_state('online', None, transport='ws')
logger.info('Connect relay: online via ws (instance %s)',
@@ -871,6 +879,13 @@ class RelayClient:
return f'drop:{_classify_ws_failure(exc)}'
finally:
self._ws = None
+ for batch in reversed(list(self._logs_inflight.values())):
+ self._logs.requeue(batch)
+ self._logs_inflight.clear()
+ self._inventory_inflight.clear()
+ self._diagnostics_inflight.clear()
+ self._diagnostics_collect = False
+ self._logs.sources = None
try:
ws.close()
except Exception:
@@ -985,6 +1000,8 @@ class RelayClient:
batch = self._logs_inflight.pop(stream_id, None)
payload = frame.get('p') or {}
self._logs.apply_ack(payload)
+ if not self._logs.collect:
+ self._logs_inflight.clear()
if not payload.get('ok') and self._logs.collect:
# Refused for a reason that is not "stop": the lines are
# still wanted and their ids make the resend safe.
@@ -1104,15 +1121,6 @@ class RelayClient:
# -- the inventory and diagnostics streams ----------
- def publish_inventory(self) -> bool:
- """Send one inventory document now. Used after a deploy, a rename
- or a removal so Cloud does not wait five minutes to learn of it."""
- ws = self._ws
- if ws is None:
- return False
- self._inventory_next_at = 0.0
- return self._publish_inventory(ws)
-
def _publish_inventory(self, ws) -> bool:
now = time.monotonic()
if now < self._inventory_next_at:
@@ -1126,8 +1134,12 @@ class RelayClient:
return False
self._inventory_stream += 1
stream_id = f'inv{self._inventory_stream}'
+ self._inventory_inflight.clear()
self._inventory_inflight[stream_id] = doc.get('generation')
- return self._send(ws, connect_inventory.inventory_frame(stream_id, doc))
+ sent = self._send(ws, connect_inventory.inventory_frame(stream_id, doc))
+ if not sent:
+ self._inventory_inflight.pop(stream_id, None)
+ return sent
def _apply_inventory_ack(self, payload: dict):
"""Cloud's answer carries the interval it wants; a refusal is logged
@@ -1147,23 +1159,38 @@ class RelayClient:
return False
self._diagnostics_next_at = now + DIAGNOSTICS_INTERVAL_S
try:
- doc = connect_inventory.build_diagnostics(int(DIAGNOSTICS_INTERVAL_S))
+ if self._diagnostics_collect:
+ doc = connect_inventory.build_diagnostics(int(DIAGNOSTICS_INTERVAL_S))
+ else:
+ doc = {'schema': connect_inventory.DIAGNOSTICS_SCHEMA,
+ 'observed_at': datetime.now(timezone.utc).isoformat(),
+ 'interval_s': int(DIAGNOSTICS_INTERVAL_S),
+ 'processes': [], 'mounts': [], 'completeness': {}}
except Exception:
logger.debug('Connect diagnostics: could not build the document', exc_info=True)
return False
self._diagnostics_stream += 1
stream_id = f'dia{self._diagnostics_stream}'
+ self._diagnostics_inflight.clear()
self._diagnostics_inflight[stream_id] = True
- return self._send(ws, connect_inventory.inventory_frame(stream_id, doc))
+ sent = self._send(ws, connect_inventory.inventory_frame(stream_id, doc))
+ if not sent:
+ self._diagnostics_inflight.pop(stream_id, None)
+ return sent
def _apply_diagnostics_ack(self, payload: dict):
"""`collect: false` is Cloud saying the consent is not held. Stop, and
try again slowly so a consent granted later takes effect."""
- if payload.get('ok') and payload.get('collect', True):
+ if payload.get('ok') and payload.get('collect') is True:
+ newly_allowed = not self._diagnostics_collect
+ self._diagnostics_collect = True
interval = payload.get('next_interval_s')
if isinstance(interval, (int, float)) and 30 <= interval <= 3600:
self._diagnostics_next_at = time.monotonic() + float(interval)
+ if newly_allowed:
+ self._diagnostics_next_at = 0.0
return
+ self._diagnostics_collect = False
self._diagnostics_next_at = time.monotonic() + DIAGNOSTICS_REFUSED_RETRY_S
# -- the logs stream ---------------------------------
@@ -1173,6 +1200,15 @@ class RelayClient:
one is due. Before Cloud has answered once, the first batch is
empty and its answer is the configuration."""
self._logs.retry_after_refusal()
+ if self._logs_inflight:
+ if time.monotonic() < self._logs_ack_deadline:
+ return False
+ for batch in self._logs_inflight.values():
+ self._logs.requeue(batch)
+ self._logs_inflight.clear()
+ self._logs.sources = None
+ if not self._logs.due():
+ return False
try:
self._logs.collect_now()
except Exception:
@@ -1183,6 +1219,7 @@ class RelayClient:
self._logs_stream += 1
stream_id = f'log{self._logs_stream}'
self._logs_inflight[stream_id] = batch
+ self._logs_ack_deadline = time.monotonic() + 60.0
sent = self._send(ws, connect_logs.logs_frame(stream_id, batch))
if not sent:
self._logs_inflight.pop(stream_id, None)
diff --git a/backend/app/services/connect_inventory.py b/backend/app/services/connect_inventory.py
index 7aaff429..ba1bd4ee 100644
--- a/backend/app/services/connect_inventory.py
+++ b/backend/app/services/connect_inventory.py
@@ -120,8 +120,9 @@ def build_inventory(app=None, generation: int = 0) -> dict:
def _reason(exc) -> str:
- text = f'{type(exc).__name__}: {exc}'.strip()
- return text[:200]
+ # Driver exceptions can contain SQL parameters or connection credentials.
+ # The full exception is logged locally by the collector.
+ return type(exc).__name__
def _host() -> dict | None:
@@ -191,7 +192,7 @@ def _docker_version() -> str | None:
def _applications():
from app.models.application import Application
- rows = (Application.query
+ rows = (Application.query_active()
.order_by(Application.id.asc())
.limit(MAX_APPLICATIONS + 1)
.all())
@@ -267,7 +268,7 @@ def _source(row) -> dict | None:
def _repo_slug(url) -> str | None:
if not url:
return None
- text = str(url).strip()
+ text = str(url).strip().split('?', 1)[0].split('#', 1)[0]
# git@host:owner/name.git or https://host/owner/name(.git)
if ':' in text and '://' not in text:
text = text.split(':', 1)[1]
@@ -287,7 +288,8 @@ def _app_domains(row) -> list:
domains = row.live_domains
except Exception: # noqa: BLE001
domains = []
- for d in list(domains)[:MAX_DOMAINS_PER_APP]:
+ domains = sorted(domains, key=lambda d: not getattr(d, 'is_primary', False))
+ for d in domains[:MAX_DOMAINS_PER_APP]:
entry = {'host': d.name}
if getattr(d, 'ssl_enabled', False) and getattr(d, 'ssl_expires_at', None):
entry['tls_expires_at'] = _utc(d.ssl_expires_at)
@@ -329,7 +331,7 @@ def _admin_url(row, domains) -> str | None:
break
if not primary:
return None
- scheme = 'https' if any(d.get('tls_expires_at') for d in domains) else 'http'
+ scheme = 'https' if domains[0].get('state') == 'active' else 'http'
return f'{scheme}://{primary}/wp-admin/'
@@ -361,7 +363,7 @@ def _deployments():
out = []
complete = True
- app_ids = [r.id for r in Application.query.with_entities(Application.id)
+ app_ids = [r.id for r in Application.query_active().with_entities(Application.id)
.order_by(Application.id.asc()).limit(MAX_APPLICATIONS).all()]
for app_id in app_ids:
rows = (Deployment.query.filter_by(app_id=app_id)
@@ -413,7 +415,7 @@ def _databases():
and a host; neither is read here."""
from app.models.managed_database import ManagedDatabase
- rows = (ManagedDatabase.query
+ rows = (ManagedDatabase.query_active()
.order_by(ManagedDatabase.id.asc())
.limit(MAX_DATABASES + 1).all())
complete = len(rows) <= MAX_DATABASES
diff --git a/backend/app/services/connect_logs.py b/backend/app/services/connect_logs.py
index 9d4180be..6b3171ae 100644
--- a/backend/app/services/connect_logs.py
+++ b/backend/app/services/connect_logs.py
@@ -28,7 +28,10 @@ import json
import logging
import os
import re
+import stat
import time
+import uuid
+from contextlib import contextmanager
from datetime import datetime, timezone
from app import paths
@@ -60,10 +63,26 @@ _SEVERITY = re.compile(
# Redacted locally, so a secret never crosses the relay at all; Cloud runs
# its own, wider set on arrival.
+_SECRET_KEY = (r'(?:[a-z0-9]{1,32}[_-]){0,4}(?:pass(?:word|wd)?|secret|token|'
+ r'api[_-]?key|access[_-]?key|private[_-]?key|authorization|credentials?)')
+_QUOTED_SECRET = re.compile(
+ rf'''(?i)(?P<prefix>(?P<quote>["'])(?:{_SECRET_KEY})(?P=quote)\s*:\s*)'''
+ r'''(?P<value>"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|[^\s,}\]\[{]+)''')
+
+
+def _redact_quoted_secret(match):
+ # Preserve quotes (and JSON validity), including escaped quotes inside
+ # values. A bare scalar becomes a JSON string containing the marker.
+ value = match.group('value')
+ quote = value[0] if value[0] in ('"', "'") else '"'
+ return f'{match.group("prefix")}{quote}[redacted]{quote}'
+
+
_REDACTIONS = (
- (re.compile(r'(?i)\b(bearer|basic)\s+[A-Za-z0-9._~+/=-]{8,}'), r'\1 [redacted]'),
- (re.compile(r'(?i)\b(pass(?:word|wd)?|secret|token|api[_-]?key|access[_-]?key|'
- r'private[_-]?key|authorization|credential)\b(\s*[:=]\s*)(\"[^\"]*\"|\'[^\']*\'|\S+)'),
+ (_QUOTED_SECRET, _redact_quoted_secret),
+ (re.compile(r'''(?i)\b(bearer|basic)\s+[^\s"',}\]]+'''), r'\1 [redacted]'),
+ (re.compile(rf'(?i)\b({_SECRET_KEY})\b(\s*[:=]\s*)'
+ r'''("(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\S+)'''),
r'\1\2[redacted]'),
(re.compile(r'(?i)\b([a-z][a-z0-9+.-]*://)([^/\s:@]+):([^/\s@]+)@'), r'\1\2:[redacted]@'),
)
@@ -87,7 +106,7 @@ def _now_iso() -> str:
def record_id(source: str, generation: int, offset) -> str:
"""Stable per device: the same line at the same place is the same id."""
raw = f'{source}|{generation}|{offset}'.encode('utf-8', 'replace')
- return hashlib.sha1(raw).hexdigest()[:40]
+ return hashlib.sha256(raw).hexdigest()[:40]
def _bounded_line(text: str) -> tuple[str, bool]:
@@ -100,6 +119,26 @@ def _bounded_line(text: str) -> tuple[str, bool]:
# ==================== file tailers ====================
+@contextmanager
+def _open_log_file(path):
+ """Open a regular file without following a replaced/symlinked leaf."""
+ before = os.lstat(path)
+ if not stat.S_ISREG(before.st_mode):
+ raise OSError('log source is not a regular file')
+ flags = os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_NONBLOCK', 0)
+ flags |= getattr(os, 'O_BINARY', 0)
+ fd = os.open(path, flags)
+ try:
+ opened = os.fstat(fd)
+ if (not stat.S_ISREG(opened.st_mode)
+ or (before.st_dev, before.st_ino) != (opened.st_dev, opened.st_ino)):
+ raise OSError('log source changed while opening')
+ with os.fdopen(fd, 'rb', closefd=False) as fh:
+ yield fh, opened
+ finally:
+ os.close(fd)
+
+
class FileTail:
"""Follow one file from where it was left. Rotation or truncation
increments `generation`; the offset restarts at zero. Never reads more
@@ -112,15 +151,19 @@ class FileTail:
self.offset = 0
self.inode = None
self.missing_logged = False
+ self._discarding_line = False
def read(self) -> list[dict]:
try:
- st = os.stat(self.path)
+ with _open_log_file(self.path) as (fh, st):
+ return self._read_file(fh, st)
except OSError:
if not self.missing_logged:
logger.debug('Connect logs: %s has no file to read', self.source)
self.missing_logged = True
return []
+
+ def _read_file(self, fh, st):
self.missing_logged = False
inode = getattr(st, 'st_ino', None)
if self.inode is None:
@@ -130,18 +173,25 @@ class FileTail:
if inode != self.inode or st.st_size < self.offset:
self.generation += 1
self.inode, self.offset = inode, 0
+ self._discarding_line = False
if st.st_size == self.offset:
return []
out = []
- try:
- with open(self.path, 'rb') as fh:
- fh.seek(self.offset)
- chunk = fh.read(MAX_READ_BYTES)
- except OSError:
- return []
+ fh.seek(self.offset)
+ chunk = fh.read(MAX_READ_BYTES)
+ if self._discarding_line:
+ # Never publish the remainder of an oversized line as a new
+ # record: it may contain a secret whose key was in an earlier read.
+ newline = chunk.find(b'\n')
+ if newline < 0:
+ self.offset += len(chunk)
+ return []
+ self.offset += newline + 1
+ chunk = chunk[newline + 1:]
+ self._discarding_line = False
lines = chunk.split(b'\n')
# A partial last line waits for its newline.
- tail = lines.pop() if not chunk.endswith(b'\n') else b''
+ tail = lines.pop()
consumed = len(chunk) - len(tail)
position = self.offset
for raw in lines:
@@ -149,6 +199,14 @@ class FileTail:
if text.strip():
out.append(self._record(position, text))
position += len(raw) + 1
+ if len(tail) == MAX_READ_BYTES:
+ # No newline fits in one bounded read. Emit a marker once and
+ # advance until the newline arrives, without buffering the line.
+ rec = self._record(position, '[oversized log line omitted]')
+ rec['truncated'] = True
+ out.append(rec)
+ consumed += len(tail)
+ self._discarding_line = True
self.offset += consumed
return out
@@ -171,7 +229,7 @@ class FileTail:
def _panel_log_path() -> str | None:
"""The panel's own log file, if one is written."""
candidates = [os.path.join(paths.SERVERKIT_LOG_DIR, name)
- for name in ('serverkit.log', 'app.log', 'panel.log')]
+ for name in ('error.log', 'serverkit.log', 'app.log', 'panel.log')]
for path in candidates:
if os.path.isfile(path):
return path
@@ -184,42 +242,71 @@ def _panel_log_path() -> str | None:
class AppOutput:
"""Recent output of every managed application, read through the same
services the panel's own Logs tab uses. Lines are identified by their
- content and time rather than an offset, so the id is stable across
- overlapping reads and Cloud counts the overlap as duplicates."""
+ timestamped content and occurrence rather than message text alone.
+ Overlapping snapshots are skipped; new occurrences receive distinct ids."""
def __init__(self, app=None):
self.app = app
- self.since = time.time()
+ self.generation = uuid.uuid4().hex
+ self.sequence = 0
+ self._snapshots = {}
+ self._files = {}
def read(self) -> list[dict]:
from app.services.log_service import LogService
out = []
- since = datetime.fromtimestamp(self.since, tz=timezone.utc)
- self.since = time.time()
- for row in self._applications():
+ rows = self._applications()
+ active = {row.id for row in rows}
+ self._snapshots = {k: v for k, v in self._snapshots.items() if k in active}
+ self._files = {k: v for k, v in self._files.items() if k in active}
+ for row in rows:
lines = []
unit = getattr(row, 'systemd_unit', None)
managed = str(getattr(row, 'managed_by', '') or '')
try:
if unit:
result = LogService.get_journalctl_logs(
- unit=unit, lines=MAX_APP_LINES,
- since=since.strftime('%Y-%m-%d %H:%M:%S'))
+ unit=unit, lines=MAX_APP_LINES, precise=True)
elif managed == 'docker_compose' or getattr(row, 'app_type', '') == 'docker':
app_dir = os.path.join(paths.APPS_DIR, row.name)
result = LogService.get_docker_app_logs(row.name, app_dir, MAX_APP_LINES,
- getattr(row, 'compose_file', None))
+ getattr(row, 'compose_file', None),
+ timestamps=True)
else:
- result = LogService.get_app_logs(row.name, 'error', MAX_APP_LINES)
+ reader = self._files.get(row.id)
+ path = f'/var/log/nginx/{row.name}.error.log'
+ if reader is None or reader.path != path:
+ reader = FileTail('app.output', path)
+ self._files[row.id] = reader
+ for rec in reader.read():
+ rec['id'] = record_id('app.output', self.generation,
+ f'{row.id}:{rec["id"]}')
+ rec['app_key'] = str(row.id)
+ out.append(rec)
+ continue
if result.get('success'):
lines = [l for l in (result.get('lines') or []) if l and l.strip()]
+ else:
+ continue
except Exception: # noqa: BLE001
logger.debug('Connect logs: could not read output for %s', row.name, exc_info=True)
continue
- for text in lines[-MAX_APP_LINES:]:
+ lines = lines[-MAX_APP_LINES:]
+ # Keep fingerprints, never unredacted text, between polls.
+ snapshot = [hashlib.sha256(line.encode('utf-8', 'replace')).digest()
+ for line in lines]
+ previous = self._snapshots.get(row.id, [])
+ overlap = 0
+ for size in range(min(len(previous), len(snapshot)), 0, -1):
+ if previous[-size:] == snapshot[:size]:
+ overlap = size
+ break
+ self._snapshots[row.id] = snapshot
+ for text in lines[overlap:]:
+ self.sequence += 1
text, truncated = _bounded_line(redact(text))
rec = {
- 'id': record_id('app.output', row.id, hashlib.sha1(text.encode('utf-8', 'replace')).hexdigest()[:16]),
+ 'id': record_id('app.output', self.generation, f'{row.id}:{self.sequence}'),
'source': 'app.output',
'app_key': str(row.id),
'observed_at': _now_iso(),
@@ -236,36 +323,45 @@ class AppOutput:
from app.models.application import Application
if self.app is not None:
with self.app.app_context():
- return list(Application.query.order_by(Application.id.asc()).all())
- return list(Application.query.order_by(Application.id.asc()).all())
+ return list(Application.query_active().order_by(Application.id.asc()).all())
+ return list(Application.query_active().order_by(Application.id.asc()).all())
except Exception: # noqa: BLE001
return []
class DeployOutput:
- """The build log kept with each deployment, sent once per deployment.
+ """The build log kept with each deployment, drained in bounded pages.
Only the lines; the file's path stays here."""
def __init__(self, app=None):
self.app = app
- self.sent: set[int] = set()
+ self.offsets: dict[int, int] = {}
def read(self) -> list[dict]:
out = []
- for dep in self._recent():
- if dep.id in self.sent:
- continue
+ recent = self._recent()
+ active = {dep.id for dep in recent}
+ self.offsets = {k: v for k, v in self.offsets.items() if k in active}
+ for dep in recent:
path = getattr(dep, 'build_log_path', None)
if not path or not os.path.isfile(path):
continue
try:
- with open(path, 'r', encoding='utf-8', errors='replace') as fh:
- data = json.load(fh)
+ with _open_log_file(path) as (fh, _):
+ raw = fh.read(OUTAGE_BUFFER_BYTES + 1)
+ if len(raw) > OUTAGE_BUFFER_BYTES:
+ data = {'logs': ['[oversized deployment log omitted]']}
+ else:
+ data = json.loads(raw.decode('utf-8', 'replace'))
lines = data.get('logs') if isinstance(data, dict) else data
except (OSError, ValueError):
continue
- self.sent.add(dep.id)
- for offset, entry in enumerate((lines or [])[:MAX_BATCH_RECORDS]):
+ if not isinstance(lines, list):
+ continue
+ start = self.offsets.get(dep.id, 0)
+ end = min(len(lines), start + MAX_BATCH_RECORDS - len(out))
+ for offset in range(start, end):
+ entry = lines[offset]
text = entry.get('message') if isinstance(entry, dict) else str(entry)
if not text or not str(text).strip():
continue
@@ -281,15 +377,23 @@ class DeployOutput:
}
if truncated:
rec['truncated'] = True
+ if len(raw) > OUTAGE_BUFFER_BYTES:
+ rec['truncated'] = True
out.append(rec)
+ self.offsets[dep.id] = end
+ if len(out) >= MAX_BATCH_RECORDS:
+ break
return out
def _recent(self):
try:
from app.models.deployment import Deployment
+ from app.models.application import Application
def query():
- return list(Deployment.query.order_by(Deployment.id.desc()).limit(20).all())
+ return list(Deployment.query.join(Application, Deployment.app_id == Application.id)
+ .filter(Application.deleted_at.is_(None))
+ .order_by(Deployment.id.desc()).limit(20).all())
if self.app is not None:
with self.app.app_context():
return query()
@@ -335,6 +439,7 @@ class LogPublisher:
records were added."""
if not self.collect or time.monotonic() < self.refused_until:
return 0
+ self._expire()
added = 0
for source in self.wanted():
try:
@@ -369,18 +474,23 @@ class LogPublisher:
# -- batches -----------------------------------------------------------
def due(self) -> bool:
+ self._expire()
if not self.collect or time.monotonic() < self.refused_until:
return False
if time.monotonic() < self.next_flush_at:
return False
# Before Cloud has said which sources it wants, one empty batch asks.
- return bool(self.pending) or self.sources is None
+ # Empty probes also refresh source selection while the server is quiet.
+ return True
def take_batch(self) -> dict:
"""The next batch within the bounds, removed from the buffer."""
+ self._expire()
self.next_flush_at = time.monotonic() + self.flush_interval
+ queued_at = self.oldest_pending_at
batch, size = [], 0
- while self.pending and len(batch) < MAX_BATCH_RECORDS:
+ # Reconnects negotiate consent with an empty probe before retrying data.
+ while self.sources is not None and self.pending and len(batch) < MAX_BATCH_RECORDS:
rec = self.pending[0]
rec_size = len(json.dumps(rec).encode('utf-8'))
if batch and size + rec_size > MAX_BATCH_BYTES - 1024:
@@ -391,25 +501,59 @@ class LogPublisher:
if not self.pending:
self.oldest_pending_at = None
self.generation += 1
- return {'schema': LOGS_SCHEMA, 'generation': self.generation, 'records': batch}
+ return {'schema': LOGS_SCHEMA, 'generation': self.generation, 'records': batch,
+ '_queued_at': queued_at}
def requeue(self, batch: dict):
"""A batch that was sent and not acknowledged goes back to the front;
its ids make the resend safe."""
+ self._expire()
+ if not self.collect:
+ return
+ now = time.monotonic()
+ queued_at = (batch or {}).get('_queued_at')
+ if queued_at is None:
+ queued_at = now
+ if now - queued_at > OUTAGE_BUFFER_S:
+ return
records = list((batch or {}).get('records') or [])
- self.pending[:0] = records
- self.pending_bytes += sum(len(json.dumps(r).encode('utf-8')) for r in records)
- if self.pending and self.oldest_pending_at is None:
- self.oldest_pending_at = time.monotonic()
+ pending, size, seen = [], 0, set()
+ for rec in records + self.pending:
+ if rec['id'] in seen:
+ continue
+ if self.sources is not None and rec['source'] not in self.wanted():
+ continue
+ rec_size = len(json.dumps(rec).encode('utf-8'))
+ if size + rec_size > OUTAGE_BUFFER_BYTES:
+ break
+ pending.append(rec)
+ size += rec_size
+ seen.add(rec['id'])
+ self.pending, self.pending_bytes = pending, size
+ if pending:
+ self.oldest_pending_at = min(queued_at, self.oldest_pending_at
+ if self.oldest_pending_at is not None else now)
+ else:
+ self.oldest_pending_at = None
def apply_ack(self, payload: dict):
"""Cloud's answer is the configuration."""
sources = payload.get('sources')
if isinstance(sources, list):
- self.sources = [s for s in sources if isinstance(s, str)]
+ previous = set(self.wanted())
+ self.sources = list(dict.fromkeys(s for s in sources
+ if isinstance(s, str) and s in SOURCES))
+ self.pending = [r for r in self.pending if r['source'] in self.wanted()]
+ self.pending_bytes = sum(len(json.dumps(r).encode('utf-8')) for r in self.pending)
+ if not self.pending:
+ self.oldest_pending_at = None
+ # Resume a disabled source from a fresh cursor, never backfill the
+ # period for which collection was not authorized.
+ for source in previous - set(self.wanted()):
+ self._reset_reader(source)
# `collect` is the instruction when it is present; a refusal that
# does not carry one (an unknown device, say) is also a stop.
- stop = not payload.get('collect', bool(payload.get('ok')))
+ stop = payload.get('collect') is not True
if stop:
self.collect = False
self.refused_until = time.monotonic() + REFUSED_RETRY_S
@@ -419,14 +563,26 @@ class LogPublisher:
self.pending.clear()
self.pending_bytes = 0
self.oldest_pending_at = None
+ for source in self._readers:
+ self._reset_reader(source)
logger.info('Connect logs: Cloud is not collecting (%s)', payload.get('reason') or 'off')
return
self.collect = True
self.refused_until = 0.0
interval = payload.get('flush_interval_s')
- if isinstance(interval, (int, float)) and 1 <= interval <= 300:
+ if isinstance(interval, (int, float)) and FLUSH_INTERVAL_S <= interval <= 300:
self.flush_interval = float(interval)
+ def _reset_reader(self, source):
+ if source == 'serverkit.service':
+ self._readers[source] = FileTail(source, _panel_log_path())
+ elif source == 'web.error':
+ self._readers[source] = FileTail(source, WEB_ERROR_LOG)
+ elif source == 'app.output':
+ self._readers[source] = AppOutput(self.app)
+ elif source == 'deploy.output':
+ self._readers[source] = DeployOutput(self.app)
+
def retry_after_refusal(self):
"""Called on the slow retry: ask again with one empty batch."""
if not self.collect and time.monotonic() >= self.refused_until:
@@ -435,4 +591,6 @@ class LogPublisher:
def logs_frame(stream_id: str, batch: dict) -> dict:
- return {'s': stream_id, 't': 'open', 'k': 'logs', 'p': batch}
+ # Queue age belongs to the local retry buffer, never the wire contract.
+ payload = {key: batch[key] for key in ('schema', 'generation', 'records')}
+ return {'s': stream_id, 't': 'open', 'k': 'logs', 'p': payload}
diff --git a/backend/app/services/log_service.py b/backend/app/services/log_service.py
index 42e92bda..94aa9968 100644
--- a/backend/app/services/log_service.py
+++ b/backend/app/services/log_service.py
@@ -217,13 +217,15 @@ class LogService:
@classmethod
def get_docker_app_logs(cls, app_name: str, app_dir: str, lines: int = 100,
- compose_file: str = None) -> Dict:
+ compose_file: str = None, timestamps: bool = False) -> Dict:
"""Get logs for a Docker Compose application."""
try:
cmd = ['docker', 'compose']
if compose_file:
cmd.extend(['-f', compose_file])
cmd.extend(['logs', '--tail', str(lines), '--no-color'])
+ if timestamps:
+ cmd.append('--timestamps')
result = run_checked(cmd, timeout=30, cwd=app_dir)
@@ -236,6 +238,8 @@ class LogService:
if compose_file:
cmd.extend(['-f', compose_file])
cmd.extend(['logs', '--tail', str(lines), '--no-color'])
+ if timestamps:
+ cmd.append('--timestamps')
result = run_checked(cmd, timeout=30, cwd=app_dir)
if result['success']:
log_lines = result['output'].split('\n') if result['output'] else []
@@ -251,10 +255,11 @@ class LogService:
@classmethod
def get_journalctl_logs(cls, unit: str = None, lines: int = 100,
- since: str = None, priority: str = None) -> Dict:
+ since: str = None, priority: str = None,
+ precise: bool = False) -> Dict:
"""Get system logs, trying journalctl → syslog → Windows Event Log."""
if is_command_available('journalctl'):
- return cls._read_journalctl(unit, lines, since, priority)
+ return cls._read_journalctl(unit, lines, since, priority, precise)
syslog_path = cls._find_syslog()
if syslog_path:
@@ -266,10 +271,12 @@ class LogService:
return {'success': False, 'error': 'No system log source available — journalctl, syslog, and Windows Event Log are all unavailable'}
@classmethod
- def _read_journalctl(cls, unit: str, lines: int, since: str, priority: str) -> Dict:
+ def _read_journalctl(cls, unit: str, lines: int, since: str, priority: str,
+ precise: bool = False) -> Dict:
"""Read logs from systemd journal."""
try:
- cmd = ['journalctl', '-n', str(lines), '--no-pager', '-o', 'short-iso']
+ output = 'short-iso-precise' if precise else 'short-iso'
+ cmd = ['journalctl', '-n', str(lines), '--no-pager', '-o', output]
if unit:
cmd.extend(['-u', unit])
diff --git a/backend/tests/test_connect_inventory.py b/backend/tests/test_connect_inventory.py
index f0319901..b38b15a1 100644
--- a/backend/tests/test_connect_inventory.py
+++ b/backend/tests/test_connect_inventory.py
@@ -9,14 +9,15 @@ import json
from datetime import datetime, timedelta
import pytest
+from types import SimpleNamespace
from app.services import connect_inventory
+from factories import make_application, make_user
@pytest.fixture()
def apps(app):
from app import db
- from app.models.application import Application
from app.models.deployment import Deployment
from app.models.domain import Domain
from app.models.user import User
@@ -24,17 +25,13 @@ def apps(app):
with app.app_context():
user = User.query.first()
if user is None:
- user = User(username='inv', email='[email protected]')
- if hasattr(user, 'set_password'):
- user.set_password('x' * 12)
- db.session.add(user)
- db.session.flush()
- shop = Application(name='shop', app_type='wordpress', status='running',
- port=8080, user_id=user.id)
- worker = Application(name='worker', app_type='docker', status='error',
- docker_image='registry.test/worker:1', user_id=user.id)
- db.session.add_all([shop, worker])
- db.session.flush()
+ user = make_user(db)
+ shop = make_application(db, name='shop', app_type='wordpress', status='running',
+ port=8080, user_id=user.id, managed_by=None,
+ compose_file=None, docker_image=None)
+ worker = make_application(db, name='worker', app_type='docker', status='error',
+ docker_image='registry.test/worker:1', user_id=user.id,
+ managed_by=None, compose_file=None)
db.session.add(Domain(name='shop.example.test', is_primary=True, ssl_enabled=True,
ssl_expires_at=datetime.utcnow() + timedelta(days=30),
application_id=shop.id))
@@ -120,7 +117,7 @@ def test_a_collector_that_raises_leaves_its_section_out_and_says_why(app, apps,
assert 'databases' not in doc
assert 'databases' not in doc['capabilities']
assert doc['sections']['databases']['complete'] is False
- assert 'docker did not answer' in doc['sections']['databases']['error']
+ assert doc['sections']['databases']['error'] == 'RuntimeError'
# The other sections are still there.
assert 'applications' in doc['capabilities']
@@ -156,3 +153,89 @@ def test_the_client_backs_off_when_cloud_refuses_diagnostics():
assert client._diagnostics_next_at > time.monotonic() + connect_client.DIAGNOSTICS_INTERVAL_S
client._apply_inventory_ack({'ok': True, 'next_interval_s': 300})
assert client._inventory_next_at > time.monotonic() + 200
+
+
+ 'https://oauth2:[email protected]/acme/api.git?token=secret#private',
+ '[email protected]:acme/api.git?token=secret#private',
+ 'https://github.com/acme/api.git#token=secret',
+])
+def test_repo_slugs_strip_queries_and_fragments(url):
+ assert connect_inventory._repo_slug(url) == 'acme/api'
+
+
+def test_admin_link_uses_primary_domains_own_tls_state():
+ secondary = SimpleNamespace(name='secondary.test', is_primary=False, ssl_enabled=True,
+ ssl_expires_at=datetime.utcnow() + timedelta(days=30))
+ primary = SimpleNamespace(name='primary.test', is_primary=True, ssl_enabled=False)
+ row = SimpleNamespace(app_type='wordpress', live_domains=[secondary, primary])
+ domains = connect_inventory._app_domains(row)
+ assert connect_inventory._admin_url(row, domains) == 'http://primary.test/wp-admin/'
+ primary.ssl_enabled = True
+ domains = connect_inventory._app_domains(row)
+ assert connect_inventory._admin_url(row, domains) == 'https://primary.test/wp-admin/'
+
+
+def test_deleted_resources_and_their_logs_are_excluded(app, apps):
+ from app import db
+ from app.models.application import Application
+ from app.models.managed_database import ManagedDatabase
+ from app.services.connect_logs import AppOutput, DeployOutput
+
+ with app.app_context():
+ shop = db.session.get(Application, apps['shop'])
+ shop.soft_delete()
+ active = ManagedDatabase(name='active-db', engine='postgresql')
+ deleted = ManagedDatabase(name='deleted-db', engine='postgresql')
+ deleted.soft_delete()
+ db.session.add_all([active, deleted])
+ db.session.commit()
+ doc = connect_inventory.build_inventory(app)
+ assert str(shop.id) not in {r['key'] for r in doc['applications']}
+ assert str(shop.id) not in {r['app_key'] for r in doc['deployments']}
+ assert [r['name'] for r in doc['databases']] == ['active-db']
+ assert shop.id not in {r.id for r in AppOutput(app)._applications()}
+ assert shop.id not in {r.app_id for r in DeployOutput(app)._recent()}
+
+
[email protected]('consent', [None, False, 1, 'true'])
+def test_diagnostics_probes_never_collect_without_explicit_consent(monkeypatch, consent):
+ from app.services import connect_client
+ client = connect_client.RelayClient()
+
+ def forbidden(*args):
+ pytest.fail('processes and mounts must not be collected without consent')
+
+ monkeypatch.setattr(connect_inventory, 'build_diagnostics', forbidden)
+ frames = []
+ ws = SimpleNamespace(send=lambda raw: frames.append(json.loads(raw)))
+ assert client._publish_diagnostics(ws)
+ assert frames[-1]['p']['processes'] == []
+ assert frames[-1]['p']['mounts'] == []
+ payload = {'ok': True}
+ if consent is not None:
+ payload['collect'] = consent
+ client._apply_diagnostics_ack(payload)
+ client._diagnostics_next_at = 0.0 # the refusal retry window has elapsed
+ assert client._publish_diagnostics(ws)
+ assert frames[-1]['p']['processes'] == []
+ assert client._diagnostics_collect is False
+
+
+def test_explicit_diagnostics_consent_enables_next_collection(monkeypatch):
+ from app.services import connect_client
+ client = connect_client.RelayClient()
+ frames = []
+ ws = SimpleNamespace(send=lambda raw: frames.append(json.loads(raw)))
+ client._publish_diagnostics(ws)
+ client._handle_frame(ws, json.dumps({
+ 's': frames[-1]['s'], 't': 'close', 'p': {'ok': True, 'collect': True},
+ }))
+ monkeypatch.setattr(connect_inventory, 'build_diagnostics',
+ lambda *args: {'schema': 'diagnostics/1', 'processes': [{'pid': 7}]})
+ assert client._publish_diagnostics(ws)
+ assert frames[-1]['p']['processes'] == [{'pid': 7}]
+ client._handle_frame(ws, json.dumps({
+ 's': frames[-1]['s'], 't': 'close', 'p': {'ok': False, 'collect': False},
+ }))
+ assert client._diagnostics_collect is False
diff --git a/backend/tests/test_connect_logs.py b/backend/tests/test_connect_logs.py
index fb909b1f..f4fb2049 100644
--- a/backend/tests/test_connect_logs.py
+++ b/backend/tests/test_connect_logs.py
@@ -8,6 +8,7 @@ looks like a secret crosses the relay.
"""
import json
import os
+from types import SimpleNamespace
import pytest
@@ -88,6 +89,99 @@ def test_batches_respect_every_bound(monkeypatch):
assert first['records'][0]['id'] != second['records'][0]['id']
+ 'password', 'api_key', 'API-KEY', 'token', 'authorization', 'private_key',
+])
+def test_json_secrets_are_redacted_without_changing_other_fields(key):
+ secret = 'private "quoted" value\\with\\slashes'
+ line = json.dumps({
+ 'message': 'request failed',
+ 'context': [{key: secret, 'status': 401}],
+ })
+ redacted = json.loads(connect_logs.redact(line))
+ assert redacted == {
+ 'message': 'request failed',
+ 'context': [{key: '[redacted]', 'status': 401}],
+ }
+
+
[email protected]('secret', [123456, True, None])
+def test_json_secret_scalars_remain_valid_json(secret):
+ line = json.dumps({'password': secret, 'status': 'failed'})
+ assert json.loads(connect_logs.redact(line)) == {
+ 'password': '[redacted]', 'status': 'failed',
+ }
+
+
+def test_prefixed_json_is_redacted_before_a_file_record_is_published(logfile):
+ tail = connect_logs.FileTail('serverkit.service', str(logfile))
+ tail.read()
+ line = 'ERROR request ' + json.dumps({
+ 'password': 'example-secret', 'api_key': 'example-key', 'status': 401,
+ })
+ logfile.write_text(line + '\n', newline='\n')
+ records = tail.read()
+ assert len(records) == 1
+ assert json.loads(records[0]['text'].removeprefix('ERROR request ')) == {
+ 'password': '[redacted]', 'api_key': '[redacted]', 'status': 401,
+ }
+
+
[email protected]('extra_bytes', [0, 1, connect_logs.MAX_READ_BYTES + 20])
+def test_oversized_lines_do_not_block_later_records(logfile, extra_bytes):
+ tail = connect_logs.FileTail('serverkit.service', str(logfile))
+ tail.read()
+ prefix = b'INFO before\n'
+ oversized = b'password=' + b'x' * (
+ connect_logs.MAX_READ_BYTES + extra_bytes - len(b'password='))
+ logfile.write_bytes(prefix + oversized + b'\nINFO after\n')
+
+ records = []
+ for _ in range(8):
+ records.extend(tail.read())
+ assert [r['text'] for r in records] == [
+ 'INFO before', '[oversized log line omitted]', 'INFO after',
+ ]
+ assert records[1]['truncated'] is True
+ assert records[1]['offset'] == len(prefix)
+ assert records[2]['offset'] == len(prefix) + len(oversized) + 1
+ assert records[1]['id'] == connect_logs.record_id(
+ 'serverkit.service', 0, len(prefix))
+ assert tail.offset == logfile.stat().st_size
+ assert tail.read() == []
+
+
+def test_oversized_unterminated_line_waits_for_newline_without_leaking_remainder(logfile):
+ tail = connect_logs.FileTail('serverkit.service', str(logfile))
+ tail.read()
+ logfile.write_bytes(b'password=' + b'x' * connect_logs.MAX_READ_BYTES)
+ assert tail.read()[0]['truncated'] is True
+ assert tail.read() == []
+ assert tail.offset == logfile.stat().st_size
+
+ with logfile.open('ab') as fh:
+ fh.write(b'secret-continuation')
+ assert tail.read() == []
+ with logfile.open('ab') as fh:
+ fh.write(b'\nINFO recovered\n')
+ assert [r['text'] for r in tail.read()] == ['INFO recovered']
+
+
[email protected]('rotate', [False, True])
+def test_rotation_resets_oversized_line_discarding(logfile, rotate):
+ tail = connect_logs.FileTail('serverkit.service', str(logfile))
+ tail.read()
+ logfile.write_bytes(b'x' * connect_logs.MAX_READ_BYTES)
+ assert tail.read()[0]['truncated'] is True
+ if rotate:
+ logfile.rename(logfile.with_suffix('.old'))
+ logfile.write_bytes(b'INFO fresh file\n')
+ records = tail.read()
+ assert [r['text'] for r in records] == ['INFO fresh file']
+ assert records[0]['generation'] == 1
+ assert records[0]['offset'] == 0
+
+
def test_a_refused_batch_goes_back_to_the_front_and_a_stop_clears_it():
pub = connect_logs.LogPublisher(None)
pub.apply_ack({'ok': True, 'collect': True, 'sources': ['web.error']})
@@ -144,7 +238,7 @@ def test_deploy_output_is_sent_once_per_deployment(tmp_path):
def test_the_client_wires_the_stream_and_requeues_an_unacknowledged_batch():
from app.services import connect_client
client = connect_client.RelayClient(None)
- client._logs.apply_ack({'ok': True, 'collect': True, 'sources': []})
+ client._logs.apply_ack({'ok': True, 'collect': True, 'sources': ['web.error']})
client._logs.pending = [{'id': 'a', 'source': 'web.error', 'observed_at': 'now', 'text': 'a'}]
client._logs.pending_bytes = 60
sent = []
@@ -173,3 +267,212 @@ def test_the_client_wires_the_stream_and_requeues_an_unacknowledged_batch():
'collect': False, 'sources': []}}))
assert client._logs.collect is False and client._logs.pending == []
assert client._publish_logs(WS()) is False
+
+
+ 'Authorization: Bearer abc', 'Authorization: Basic x',
+ 'client_secret=hidden refresh_token=hidden',
+ '{"client_secret":"hidden","refresh_token":"hidden"}',
+ "password='hidden\\'also-hidden'",
+])
+def test_short_credentials_and_compound_keys_are_redacted(line):
+ redacted = connect_logs.redact(line)
+ assert 'hidden' not in redacted
+ assert not redacted.endswith((' abc', ' x'))
+ assert '[redacted]' in redacted
+
+
+def test_file_tail_rejects_symlinks(logfile, tmp_path):
+ tail = connect_logs.FileTail('web.error', str(logfile))
+ tail.read()
+ target = tmp_path / 'private.txt'
+ target.write_text('private file contents\n')
+ logfile.unlink()
+ try:
+ logfile.symlink_to(target)
+ except OSError:
+ pytest.skip('symlinks require privileges on this platform')
+ assert tail.read() == []
+ assert tail.offset == 0
+
+
+def test_replacement_between_stat_and_open_is_not_read(logfile, monkeypatch):
+ tail = connect_logs.FileTail('web.error', str(logfile))
+ tail.read()
+ original_open = os.open
+
+ def replace_then_open(path, flags):
+ logfile.rename(logfile.with_suffix('.old'))
+ logfile.write_bytes(b'private replacement contents\n')
+ return original_open(path, flags)
+
+ monkeypatch.setattr(connect_logs.os, 'open', replace_then_open)
+ assert tail.read() == []
+ assert tail.offset == 0
+
+
+def test_panel_reader_uses_production_error_log(tmp_path, monkeypatch):
+ monkeypatch.setattr(connect_logs.paths, 'SERVERKIT_LOG_DIR', str(tmp_path))
+ assert connect_logs._panel_log_path() == str(tmp_path / 'error.log')
+ (tmp_path / 'error.log').write_text('')
+ (tmp_path / 'serverkit.log').write_text('')
+ assert connect_logs._panel_log_path() == str(tmp_path / 'error.log')
+
+
+def test_deployment_reader_drains_more_than_one_batch_and_new_lines(tmp_path):
+ path = tmp_path / 'build.json'
+ lines = [f'line {i}' for i in range(450)]
+ path.write_text(json.dumps({'logs': lines}))
+ reader = connect_logs.DeployOutput()
+ reader._recent = lambda: [
+ SimpleNamespace(id=7, app_id=3, version=2, build_log_path=str(path))]
+ records = reader.read() + reader.read() + reader.read()
+ assert [r['text'] for r in records] == lines
+ assert len({r['id'] for r in records}) == 450
+ assert reader.read() == []
+ path.write_text(json.dumps({'logs': lines + ['last line']}))
+ assert [r['text'] for r in reader.read()] == ['last line']
+
+
+def test_repeated_app_messages_are_distinct_and_overlaps_are_skipped(monkeypatch):
+ from app.services.log_service import LogService
+ reader = connect_logs.AppOutput()
+ reader._applications = lambda: [
+ SimpleNamespace(id=7, name='web', app_type='docker', compose_file=None)]
+ first = 'web | 2026-09-11T01:00:00.123456789Z ERROR repeated'
+ later = 'web | 2026-09-11T01:00:01.123456789Z ERROR repeated'
+ snapshots = iter([[first, first], [first, first], [first, later]])
+
+ def read_logs(*args, **kwargs):
+ assert kwargs['timestamps'] is True
+ return {'success': True, 'lines': next(snapshots)}
+
+ monkeypatch.setattr(LogService, 'get_docker_app_logs', read_logs)
+ records = reader.read()
+ assert len(records) == 2
+ assert records[0]['id'] != records[1]['id']
+ assert reader.read() == []
+ newer = reader.read()
+ assert len(newer) == 1 and newer[0]['text'] == later
+ assert newer[0]['id'] not in {r['id'] for r in records}
+
+
[email protected]('consent', [None, False, 1, 'true'])
+def test_log_consent_requires_literal_true(consent):
+ pub = connect_logs.LogPublisher()
+ pub.apply_ack({'ok': True, 'collect': True, 'sources': ['web.error']})
+ payload = {'ok': True}
+ if consent is not None:
+ payload['collect'] = consent
+ pub.apply_ack(payload)
+ assert pub.collect is False
+ assert pub.collect_now() == 0
+ assert pub.pending == []
+
+
+def test_requeue_obeys_byte_bound_and_preserves_original_expiry(monkeypatch):
+ clock = [100.0]
+ monkeypatch.setattr(connect_logs.time, 'monotonic', lambda: clock[0])
+ pub = connect_logs.LogPublisher()
+ pub.apply_ack({'ok': True, 'collect': True, 'sources': ['web.error']})
+ record = lambda key: {'id': key, 'source': 'web.error', 'text': 'x' * 100}
+ size = len(json.dumps(record('a')).encode())
+ monkeypatch.setattr(connect_logs, 'OUTAGE_BUFFER_BYTES', size * 2)
+ pub.pending = [record('a'), record('b')]
+ pub.pending_bytes = size * 2
+ pub.oldest_pending_at = clock[0]
+ batch = pub.take_batch()
+ assert '_queued_at' not in connect_logs.logs_frame('log1', batch)['p']
+ pub.pending = [record('c')]
+ pub.pending_bytes = size
+ pub.oldest_pending_at = 200.0
+ clock[0] = 200.0
+ pub.requeue(batch)
+ assert pub.pending_bytes <= size * 2
+ assert [r['id'] for r in pub.pending] == ['a', 'b']
+ retry = pub.take_batch()
+ clock[0] = 401.0
+ pub.requeue(retry)
+ assert pub.pending == []
+
+
+def test_restricted_source_selection_purges_queued_records():
+ pub = connect_logs.LogPublisher()
+ pub.apply_ack({'ok': True, 'collect': True, 'sources': ['web.error', 'app.output']})
+ pub.pending = [
+ {'id': 'a', 'source': 'app.output', 'text': 'app'},
+ {'id': 'b', 'source': 'web.error', 'text': 'web'},
+ ]
+ pub.apply_ack({'ok': True, 'collect': True, 'sources': ['web.error']})
+ assert [r['id'] for r in pub.pending] == ['b']
+ pub.requeue({'records': [{'id': 'a', 'source': 'app.output', 'text': 'app'}]})
+ assert [r['id'] for r in pub.pending] == ['b']
+
+
+def test_log_ack_timeout_retries_only_after_empty_consent_probe(monkeypatch):
+ from app.services import connect_client
+ clock = [100.0]
+ monkeypatch.setattr(connect_logs.time, 'monotonic', lambda: clock[0])
+ client = connect_client.RelayClient()
+ client._logs.apply_ack({'ok': True, 'collect': True, 'sources': ['web.error']})
+ client._logs._readers['web.error'] = SimpleNamespace(read=lambda: [])
+ client._logs.pending = [{'id': 'a', 'source': 'web.error', 'text': 'message'}]
+ client._logs.pending_bytes = 60
+ client._logs.oldest_pending_at = clock[0]
+ frames = []
+ ws = SimpleNamespace(send=lambda raw: frames.append(json.loads(raw)))
+ assert client._publish_logs(ws)
+ assert not client._publish_logs(ws)
+ clock[0] = 161.0
+ assert client._publish_logs(ws)
+ assert frames[-1]['p']['records'] == []
+ assert [r['id'] for r in client._logs.pending] == ['a']
+ assert len(client._logs_inflight) == 1
+ client._handle_frame(ws, json.dumps({
+ 's': frames[-1]['s'], 't': 'close',
+ 'p': {'ok': True, 'collect': True, 'sources': ['web.error']},
+ }))
+ clock[0] = 167.0
+ assert client._publish_logs(ws)
+ assert frames[-1]['p']['records'][0]['id'] == 'a'
+
+
+def test_socket_drop_recovers_unacknowledged_logs(monkeypatch):
+ from app.services import connect_client
+ client = connect_client.RelayClient()
+ client.running = True
+ client._logs.apply_ack({'ok': True, 'collect': True, 'sources': ['web.error']})
+ batch = {'schema': 'logs/1', 'generation': 1,
+ 'records': [{'id': 'a', 'source': 'web.error', 'text': 'message'}],
+ '_queued_at': connect_logs.time.monotonic()}
+ client._logs_inflight['log1'] = batch
+
+ def drop(**kwargs):
+ raise ConnectionError('connection dropped before ack')
+
+ ws = SimpleNamespace(recv=drop, close=lambda: None)
+ monkeypatch.setattr(client, '_ws_connect', lambda cfg: ws)
+ monkeypatch.setattr(client, '_set_state', lambda *args, **kwargs: None)
+ for name in ('_check_for_update', '_load_jwks', '_publish_storage',
+ '_publish_policy', '_publish_inventory'):
+ monkeypatch.setattr(client, name, lambda *args: None)
+ assert client._ws_session({}).startswith('drop:')
+ assert client._logs_inflight == {}
+ assert client._logs.pending == batch['records']
+ assert client._logs.sources is None
+ assert client._logs.take_batch()['records'] == []
+
+
+def test_timestamped_log_service_options_reach_commands(fake_subprocess, monkeypatch):
+ from app.services import log_service
+ fake_subprocess.script(['docker', 'compose', 'logs'], stdout='timestamp message\n')
+ result = log_service.LogService.get_docker_app_logs(
+ 'web', '.', timestamps=True)
+ assert result['success']
+ assert '--timestamps' in fake_subprocess.argv_for(['docker', 'compose'])
+
+ monkeypatch.setattr(log_service, 'is_command_available', lambda name: True)
+ fake_subprocess.script(['journalctl'], stdout='timestamp message\n')
+ result = log_service.LogService.get_journalctl_logs(unit='web.service', precise=True)
+ assert result['success']
+ assert 'short-iso-precise' in fake_subprocess.argv_for(['journalctl'])