Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

Fix gzip database backups and restores

Juan Denis76432b123170
diff --git a/backend/app/services/backup_service.py b/backend/app/services/backup_service.py
index 2ccc16ca..0db91b83 100644
--- a/backend/app/services/backup_service.py
+++ b/backend/app/services/backup_service.py
@@ -1,9 +1,7 @@
import os
import json
-import subprocess
import shutil
import tarfile
-import gzip
import tempfile
from datetime import datetime, timedelta, timezone
from typing import Dict, List, Optional
@@ -17,7 +15,9 @@ from app import paths
from app.utils import backup_crypto
from app.utils.config_store import load_json_config, save_json_config
from app.utils.formatting import format_bytes
-from app.utils.system import is_command_available, run_checked
+from app.utils.system import (
+ is_command_available, run_checked, run_from_gzip, run_to_gzip,
+)
from app.services.telemetry_service import TelemetryService, generate_correlation_id
from app.services import backup_schedule_service
@@ -200,9 +200,7 @@ class BackupService:
cmd.extend(['-h', config['host']])
cmd.append(db_name)
- with gzip.open(backup_file, 'wt') as f:
- result = run_checked(cmd, stdout=f, stderr=subprocess.PIPE,
- timeout=None)
+ result = run_to_gzip(cmd, backup_file)
if not result['success']:
return {'success': False, 'error': result['error']}
@@ -222,9 +220,7 @@ class BackupService:
cmd.extend(['-h', config['host']])
cmd.append(db_name)
- with gzip.open(backup_file, 'wt') as f:
- result = run_checked(cmd, stdout=f, stderr=subprocess.PIPE,
- env=env, timeout=None)
+ result = run_to_gzip(cmd, backup_file, env=env)
if not result['success']:
return {'success': False, 'error': result['error']}
@@ -457,8 +453,7 @@ class BackupService:
cmd.extend(['-h', host])
cmd.append(db_name)
- with gzip.open(backup_path, 'rt') as f:
- result = run_checked(cmd, stdin=f, timeout=None)
+ result = run_from_gzip(cmd, backup_path)
elif db_type == 'postgresql':
if not is_command_available('psql'):
@@ -475,8 +470,7 @@ class BackupService:
cmd.extend(['-h', host])
cmd.extend(['-d', db_name])
- with gzip.open(backup_path, 'rt') as f:
- result = run_checked(cmd, stdin=f, env=env, timeout=None)
+ result = run_from_gzip(cmd, backup_path, env=env)
else:
return {'success': False, 'error': f'Unknown database type: {db_type}'}
diff --git a/backend/app/services/db_sync_service.py b/backend/app/services/db_sync_service.py
index 4260ca67..126808ba 100644
--- a/backend/app/services/db_sync_service.py
+++ b/backend/app/services/db_sync_service.py
@@ -8,8 +8,7 @@ Supports WordPress-specific features like URL search-replace and data anonymizat
import os
import subprocess
-from app.utils.system import run_checked
-import gzip
+from app.utils.system import run_checked, run_from_gzip, run_to_gzip
import shutil
import json
import re
@@ -106,23 +105,9 @@ class DatabaseSyncService:
# Execute dump
if compress:
- # Pipe through gzip
- dump_process = subprocess.Popen(
- cmd,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE
- )
- with gzip.open(file_path, 'wb') as f:
- while True:
- chunk = dump_process.stdout.read(8192)
- if not chunk:
- break
- f.write(chunk)
-
- dump_process.wait()
- if dump_process.returncode != 0:
- error = dump_process.stderr.read().decode()
- return {'success': False, 'error': f'mysqldump failed: {error}'}
+ result = run_to_gzip(cmd, file_path)
+ if not result['success']:
+ return {'success': False, 'error': f"mysqldump failed: {result['error']}"}
else:
with open(file_path, 'w') as f:
result = run_checked(cmd, stdout=f, stderr=subprocess.PIPE,
@@ -212,9 +197,8 @@ class DatabaseSyncService:
# Handle compressed files
if file_path.endswith('.gz'):
- with gzip.open(file_path, 'rb') as f:
- # 30 min bound for large DBs
- result = run_checked(cmd, stdin=f, timeout=1800)
+ # 30 min bound for large DBs
+ result = run_from_gzip(cmd, file_path, timeout=1800)
else:
with open(file_path, 'r') as f:
result = run_checked(cmd, stdin=f, timeout=1800)
@@ -695,23 +679,10 @@ class DatabaseSyncService:
full_cmd = ['docker', 'compose', '-f', compose_path, 'exec', '-T', 'db'] + dump_cmd
if compress:
- process = subprocess.Popen(
- full_cmd,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE
- )
- with gzip.open(output_path, 'wb') as f:
- while True:
- chunk = process.stdout.read(8192)
- if not chunk:
- break
- f.write(chunk)
- process.wait()
- if process.returncode != 0:
- error = process.stderr.read().decode()
- if os.path.exists(output_path):
- os.remove(output_path)
- return {'success': False, 'error': f'Container mysqldump failed: {error}'}
+ result = run_to_gzip(full_cmd, output_path, timeout=600)
+ if not result['success']:
+ return {'success': False,
+ 'error': f"Container mysqldump failed: {result['error']}"}
else:
with open(output_path, 'w') as f:
result = run_checked(full_cmd, stdout=f, stderr=subprocess.PIPE,
@@ -772,8 +743,7 @@ class DatabaseSyncService:
import_cmd.append(db_name)
if snapshot_path.endswith('.gz'):
- with gzip.open(snapshot_path, 'rb') as f:
- result = run_checked(import_cmd, stdin=f, timeout=1800)
+ result = run_from_gzip(import_cmd, snapshot_path, timeout=1800)
else:
with open(snapshot_path, 'r') as f:
result = run_checked(import_cmd, stdin=f, timeout=1800)
diff --git a/backend/app/utils/system.py b/backend/app/utils/system.py
index df07de16..ecfa0f09 100644
--- a/backend/app/utils/system.py
+++ b/backend/app/utils/system.py
@@ -271,6 +271,84 @@ def _never_ran(error: str) -> dict:
'returncode': None, 'error': error}
+_CHUNK = 1024 * 1024
+
+
+def _streamed(cmd: List[str], pump, *, env: Optional[dict], timeout: Optional[int],
+ **popen_kwargs) -> dict:
+ """Run *cmd* while *pump(proc)* moves bytes through its pipes.
+
+ A ``gzip.open()`` handle can't be passed as ``stdin=``/``stdout=``: the
+ child gets the underlying fd and reads or writes the compressed bytes
+ raw. So the bytes are pumped through Python instead. stderr goes to a
+ temp file so a chatty child can't fill a pipe and deadlock the pump.
+ """
+ import tempfile
+ try:
+ with tempfile.TemporaryFile() as err:
+ proc = subprocess.Popen(cmd, stderr=err, env=env, **popen_kwargs)
+ try:
+ pump(proc)
+ except BrokenPipeError:
+ pass # the child exited early; its exit code says why
+ except BaseException:
+ proc.kill()
+ proc.wait()
+ raise
+ try:
+ proc.wait(timeout=timeout)
+ except subprocess.TimeoutExpired:
+ proc.kill()
+ proc.wait()
+ return _never_ran(f'timed out after {timeout}s')
+ err.seek(0)
+ stderr = err.read().decode('utf-8', 'replace')
+ except FileNotFoundError as exc:
+ return _never_ran(f'{cmd[0]} not found: {exc}')
+ except OSError as exc:
+ return _never_ran(str(exc))
+ return {
+ 'success': proc.returncode == 0,
+ 'output': '',
+ 'stderr': stderr,
+ 'returncode': proc.returncode,
+ 'error': (stderr.strip() or f'command failed (exit {proc.returncode})')
+ if proc.returncode != 0 else None,
+ }
+
+
+def run_to_gzip(cmd: List[str], dest: str, *, env: Optional[dict] = None,
+ timeout: Optional[int] = None) -> dict:
+ """Run *cmd* and gzip its stdout into *dest*; *dest* is removed on failure."""
+ import gzip
+
+ def pump(proc):
+ with gzip.open(dest, 'wb') as out:
+ shutil.copyfileobj(proc.stdout, out, _CHUNK)
+ proc.stdout.close()
+
+ result = _streamed(cmd, pump, env=env, timeout=timeout, stdout=subprocess.PIPE)
+ if not result['success'] and os.path.exists(dest):
+ os.remove(dest)
+ return result
+
+
+def run_from_gzip(cmd: List[str], src: str, *, env: Optional[dict] = None,
+ timeout: Optional[int] = None) -> dict:
+ """Run *cmd* with the decompressed contents of *src* on its stdin."""
+ import gzip
+
+ def pump(proc):
+ try:
+ with gzip.open(src, 'rb') as data:
+ shutil.copyfileobj(data, proc.stdin, _CHUNK)
+ finally:
+ proc.stdin.close()
+
+ return _streamed(cmd, pump, env=env, timeout=timeout,
+ stdin=subprocess.PIPE, stdout=subprocess.DEVNULL)
+
+
def write_privileged_file(path: str, content: str, *, append: bool = False,
mode: Optional[str] = None,
owner: Optional[str] = None) -> dict:
diff --git a/backend/tests/RAW_SUBPROCESS_CEILING b/backend/tests/RAW_SUBPROCESS_CEILING
index a45fd52c..40994076 100644
--- a/backend/tests/RAW_SUBPROCESS_CEILING
+++ b/backend/tests/RAW_SUBPROCESS_CEILING
@@ -1 +1 @@
-24
+23
diff --git a/backend/tests/test_backup_db_gzip.py b/backend/tests/test_backup_db_gzip.py
new file mode 100644
index 00000000..b48a4d03
--- /dev/null
+++ b/backend/tests/test_backup_db_gzip.py
@@ -0,0 +1,99 @@
+"""Database dumps must be real gzip, and restores must feed the client SQL.
+
+Passing a ``gzip.open()`` handle as a subprocess stream hands the child the
+raw fd: dumps came out as plain SQL inside a gzip header, and restores
+piped compressed bytes into mysql/psql. These run real child processes —
+small Python stand-ins for mysqldump/mysql — so the bytes really cross a pipe.
+"""
+import gzip
+import subprocess
+import sys
+
+import pytest
+
+from app.utils import system
+from app.utils.system import run_from_gzip, run_to_gzip
+
+_ROW = "INSERT INTO t VALUES (1), (2);\n"
+_COUNT = 40000 # ~1.2 MB: more than a pipe buffer and more than one copy chunk
+SQL = _ROW * _COUNT
+# The child builds the dump itself — Windows caps a command line at 32K.
+WRITE_SQL = f'import sys; sys.stdout.write({_ROW!r} * {_COUNT})'
+
+
+def test_run_to_gzip_writes_real_gzip(tmp_path):
+ dest = tmp_path / 'db.sql.gz'
+ result = run_to_gzip(
+ [sys.executable, '-c', WRITE_SQL], str(dest))
+ assert result['success'], result
+ with gzip.open(dest, 'rt') as f:
+ assert f.read() == SQL
+
+
+def test_run_to_gzip_failure_removes_partial_file(tmp_path):
+ dest = tmp_path / 'db.sql.gz'
+ result = run_to_gzip(
+ [sys.executable, '-c', 'import sys; print("partial"); '
+ 'sys.stderr.write("access denied"); sys.exit(2)'],
+ str(dest))
+ assert not result['success']
+ assert result['returncode'] == 2
+ assert 'access denied' in result['error']
+ assert not dest.exists()
+
+
+def test_run_from_gzip_feeds_decompressed_sql(tmp_path):
+ src, out = tmp_path / 'db.sql.gz', tmp_path / 'received.sql'
+ with gzip.open(src, 'wt') as f:
+ f.write(SQL)
+ result = run_from_gzip(
+ [sys.executable, '-c',
+ f'import sys; open({str(out)!r}, "wb").write(sys.stdin.buffer.read())'],
+ str(src))
+ assert result['success'], result
+ assert out.read_text() == SQL
+
+
+def test_run_from_gzip_child_exiting_early_reports_its_exit(tmp_path):
+ src = tmp_path / 'db.sql.gz'
+ with gzip.open(src, 'wt') as f:
+ f.write(SQL * 5)
+ result = run_from_gzip(
+ [sys.executable, '-c', 'import sys; sys.stderr.write("syntax error"); sys.exit(1)'],
+ str(src))
+ assert not result['success']
+ assert result['returncode'] == 1
+
+
+def fake_clients(tmp_path, monkeypatch):
+ """Route mysqldump/mysql to Python stand-ins; mysql records its stdin."""
+ received = tmp_path / 'mysql_stdin.sql'
+ stand_ins = {
+ 'mysqldump': WRITE_SQL,
+ 'mysql': f'import sys; open({str(received)!r}, "wb").write(sys.stdin.buffer.read())',
+ }
+ real_popen = subprocess.Popen
+
+ def popen(cmd, *args, **kwargs):
+ if cmd[0] in stand_ins:
+ cmd = [sys.executable, '-c', stand_ins[cmd[0]]]
+ return real_popen(cmd, *args, **kwargs)
+
+ monkeypatch.setattr(system.subprocess, 'Popen', popen)
+ monkeypatch.setattr('app.services.backup_service.is_command_available', lambda c: True)
+ return received
+
+
+def test_mysql_backup_then_restore_round_trips(app, tmp_path, fake_clients):
+ from app.services.backup_service import BackupService
+ backup = BackupService._backup_database_internal(
+ 'mysql', 'shop', str(tmp_path), {'user': 'root', 'password': 'pw'})
+ assert backup['success'], backup
+ with gzip.open(backup['path'], 'rt') as f:
+ assert f.read() == SQL
+
+ restored = BackupService.restore_database(backup['path'], 'mysql', 'shop',
+ user='root', password='pw')
+ assert restored['success'], restored
+ assert fake_clients.read_text() == SQL