ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
Fix gzip database backups and restores
Juan Denis76432b123170
diff --git a/backend/app/services/backup_service.py b/backend/app/services/backup_service.pyindex 2ccc16ca..0db91b83 100644--- a/backend/app/services/backup_service.py+++ b/backend/app/services/backup_service.py@@ -1,9 +1,7 @@import osimport json-import subprocessimport shutilimport tarfile-import gzipimport tempfilefrom datetime import datetime, timedelta, timezonefrom typing import Dict, List, Optional@@ -17,7 +15,9 @@ from app import pathsfrom app.utils import backup_cryptofrom app.utils.config_store import load_json_config, save_json_configfrom app.utils.formatting import format_bytes-from app.utils.system import is_command_available, run_checked+from app.utils.system import (+ is_command_available, run_checked, run_from_gzip, run_to_gzip,+)from app.services.telemetry_service import TelemetryService, generate_correlation_idfrom app.services import backup_schedule_service@@ -200,9 +200,7 @@ class BackupService:cmd.extend(['-h', config['host']])cmd.append(db_name)- with gzip.open(backup_file, 'wt') as f:- result = run_checked(cmd, stdout=f, stderr=subprocess.PIPE,- timeout=None)+ result = run_to_gzip(cmd, backup_file)if not result['success']:return {'success': False, 'error': result['error']}@@ -222,9 +220,7 @@ class BackupService:cmd.extend(['-h', config['host']])cmd.append(db_name)- with gzip.open(backup_file, 'wt') as f:- result = run_checked(cmd, stdout=f, stderr=subprocess.PIPE,- env=env, timeout=None)+ result = run_to_gzip(cmd, backup_file, env=env)if not result['success']:return {'success': False, 'error': result['error']}@@ -457,8 +453,7 @@ class BackupService:cmd.extend(['-h', host])cmd.append(db_name)- with gzip.open(backup_path, 'rt') as f:- result = run_checked(cmd, stdin=f, timeout=None)+ result = run_from_gzip(cmd, backup_path)elif db_type == 'postgresql':if not is_command_available('psql'):@@ -475,8 +470,7 @@ class BackupService:cmd.extend(['-h', host])cmd.extend(['-d', db_name])- with gzip.open(backup_path, 'rt') as f:- result = run_checked(cmd, stdin=f, env=env, timeout=None)+ result = run_from_gzip(cmd, backup_path, env=env)else:return {'success': False, 'error': f'Unknown database type: {db_type}'}diff --git a/backend/app/services/db_sync_service.py b/backend/app/services/db_sync_service.pyindex 4260ca67..126808ba 100644--- a/backend/app/services/db_sync_service.py+++ b/backend/app/services/db_sync_service.py@@ -8,8 +8,7 @@ Supports WordPress-specific features like URL search-replace and data anonymizatimport osimport subprocess-from app.utils.system import run_checked-import gzip+from app.utils.system import run_checked, run_from_gzip, run_to_gzipimport shutilimport jsonimport re@@ -106,23 +105,9 @@ class DatabaseSyncService:# Execute dumpif compress:- # Pipe through gzip- dump_process = subprocess.Popen(- cmd,- stdout=subprocess.PIPE,- stderr=subprocess.PIPE- )- with gzip.open(file_path, 'wb') as f:- while True:- chunk = dump_process.stdout.read(8192)- if not chunk:- break- f.write(chunk)-- dump_process.wait()- if dump_process.returncode != 0:- error = dump_process.stderr.read().decode()- return {'success': False, 'error': f'mysqldump failed: {error}'}+ result = run_to_gzip(cmd, file_path)+ if not result['success']:+ return {'success': False, 'error': f"mysqldump failed: {result['error']}"}else:with open(file_path, 'w') as f:result = run_checked(cmd, stdout=f, stderr=subprocess.PIPE,@@ -212,9 +197,8 @@ class DatabaseSyncService:# Handle compressed filesif file_path.endswith('.gz'):- with gzip.open(file_path, 'rb') as f:- # 30 min bound for large DBs- result = run_checked(cmd, stdin=f, timeout=1800)+ # 30 min bound for large DBs+ result = run_from_gzip(cmd, file_path, timeout=1800)else:with open(file_path, 'r') as f:result = run_checked(cmd, stdin=f, timeout=1800)@@ -695,23 +679,10 @@ class DatabaseSyncService:full_cmd = ['docker', 'compose', '-f', compose_path, 'exec', '-T', 'db'] + dump_cmdif compress:- process = subprocess.Popen(- full_cmd,- stdout=subprocess.PIPE,- stderr=subprocess.PIPE- )- with gzip.open(output_path, 'wb') as f:- while True:- chunk = process.stdout.read(8192)- if not chunk:- break- f.write(chunk)- process.wait()- if process.returncode != 0:- error = process.stderr.read().decode()- if os.path.exists(output_path):- os.remove(output_path)- return {'success': False, 'error': f'Container mysqldump failed: {error}'}+ result = run_to_gzip(full_cmd, output_path, timeout=600)+ if not result['success']:+ return {'success': False,+ 'error': f"Container mysqldump failed: {result['error']}"}else:with open(output_path, 'w') as f:result = run_checked(full_cmd, stdout=f, stderr=subprocess.PIPE,@@ -772,8 +743,7 @@ class DatabaseSyncService:import_cmd.append(db_name)if snapshot_path.endswith('.gz'):- with gzip.open(snapshot_path, 'rb') as f:- result = run_checked(import_cmd, stdin=f, timeout=1800)+ result = run_from_gzip(import_cmd, snapshot_path, timeout=1800)else:with open(snapshot_path, 'r') as f:result = run_checked(import_cmd, stdin=f, timeout=1800)diff --git a/backend/app/utils/system.py b/backend/app/utils/system.pyindex df07de16..ecfa0f09 100644--- a/backend/app/utils/system.py+++ b/backend/app/utils/system.py@@ -271,6 +271,84 @@ def _never_ran(error: str) -> dict:'returncode': None, 'error': error}+_CHUNK = 1024 * 1024+++def _streamed(cmd: List[str], pump, *, env: Optional[dict], timeout: Optional[int],+ **popen_kwargs) -> dict:+ """Run *cmd* while *pump(proc)* moves bytes through its pipes.++ A ``gzip.open()`` handle can't be passed as ``stdin=``/``stdout=``: the+ child gets the underlying fd and reads or writes the compressed bytes+ raw. So the bytes are pumped through Python instead. stderr goes to a+ temp file so a chatty child can't fill a pipe and deadlock the pump.+ """+ import tempfile+ try:+ with tempfile.TemporaryFile() as err:+ proc = subprocess.Popen(cmd, stderr=err, env=env, **popen_kwargs)+ try:+ pump(proc)+ except BrokenPipeError:+ pass # the child exited early; its exit code says why+ except BaseException:+ proc.kill()+ proc.wait()+ raise+ try:+ proc.wait(timeout=timeout)+ except subprocess.TimeoutExpired:+ proc.kill()+ proc.wait()+ return _never_ran(f'timed out after {timeout}s')+ err.seek(0)+ stderr = err.read().decode('utf-8', 'replace')+ except FileNotFoundError as exc:+ return _never_ran(f'{cmd[0]} not found: {exc}')+ except OSError as exc:+ return _never_ran(str(exc))+ return {+ 'success': proc.returncode == 0,+ 'output': '',+ 'stderr': stderr,+ 'returncode': proc.returncode,+ 'error': (stderr.strip() or f'command failed (exit {proc.returncode})')+ if proc.returncode != 0 else None,+ }+++def run_to_gzip(cmd: List[str], dest: str, *, env: Optional[dict] = None,+ timeout: Optional[int] = None) -> dict:+ """Run *cmd* and gzip its stdout into *dest*; *dest* is removed on failure."""+ import gzip++ def pump(proc):+ with gzip.open(dest, 'wb') as out:+ shutil.copyfileobj(proc.stdout, out, _CHUNK)+ proc.stdout.close()++ result = _streamed(cmd, pump, env=env, timeout=timeout, stdout=subprocess.PIPE)+ if not result['success'] and os.path.exists(dest):+ os.remove(dest)+ return result+++def run_from_gzip(cmd: List[str], src: str, *, env: Optional[dict] = None,+ timeout: Optional[int] = None) -> dict:+ """Run *cmd* with the decompressed contents of *src* on its stdin."""+ import gzip++ def pump(proc):+ try:+ with gzip.open(src, 'rb') as data:+ shutil.copyfileobj(data, proc.stdin, _CHUNK)+ finally:+ proc.stdin.close()++ return _streamed(cmd, pump, env=env, timeout=timeout,+ stdin=subprocess.PIPE, stdout=subprocess.DEVNULL)++def write_privileged_file(path: str, content: str, *, append: bool = False,mode: Optional[str] = None,owner: Optional[str] = None) -> dict:diff --git a/backend/tests/RAW_SUBPROCESS_CEILING b/backend/tests/RAW_SUBPROCESS_CEILINGindex a45fd52c..40994076 100644--- a/backend/tests/RAW_SUBPROCESS_CEILING+++ b/backend/tests/RAW_SUBPROCESS_CEILING@@ -1 +1 @@-24+23diff --git a/backend/tests/test_backup_db_gzip.py b/backend/tests/test_backup_db_gzip.pynew file mode 100644index 00000000..b48a4d03--- /dev/null+++ b/backend/tests/test_backup_db_gzip.py@@ -0,0 +1,99 @@+"""Database dumps must be real gzip, and restores must feed the client SQL.++Passing a ``gzip.open()`` handle as a subprocess stream hands the child the+raw fd: dumps came out as plain SQL inside a gzip header, and restores+piped compressed bytes into mysql/psql. These run real child processes —+small Python stand-ins for mysqldump/mysql — so the bytes really cross a pipe.+"""+import gzip+import subprocess+import sys++import pytest++from app.utils import system+from app.utils.system import run_from_gzip, run_to_gzip++_ROW = "INSERT INTO t VALUES (1), (2);\n"+_COUNT = 40000 # ~1.2 MB: more than a pipe buffer and more than one copy chunk+SQL = _ROW * _COUNT+# The child builds the dump itself — Windows caps a command line at 32K.+WRITE_SQL = f'import sys; sys.stdout.write({_ROW!r} * {_COUNT})'+++def test_run_to_gzip_writes_real_gzip(tmp_path):+ dest = tmp_path / 'db.sql.gz'+ result = run_to_gzip(+ [sys.executable, '-c', WRITE_SQL], str(dest))+ assert result['success'], result+ with gzip.open(dest, 'rt') as f:+ assert f.read() == SQL+++def test_run_to_gzip_failure_removes_partial_file(tmp_path):+ dest = tmp_path / 'db.sql.gz'+ result = run_to_gzip(+ [sys.executable, '-c', 'import sys; print("partial"); '+ 'sys.stderr.write("access denied"); sys.exit(2)'],+ str(dest))+ assert not result['success']+ assert result['returncode'] == 2+ assert 'access denied' in result['error']+ assert not dest.exists()+++def test_run_from_gzip_feeds_decompressed_sql(tmp_path):+ src, out = tmp_path / 'db.sql.gz', tmp_path / 'received.sql'+ with gzip.open(src, 'wt') as f:+ f.write(SQL)+ result = run_from_gzip(+ [sys.executable, '-c',+ f'import sys; open({str(out)!r}, "wb").write(sys.stdin.buffer.read())'],+ str(src))+ assert result['success'], result+ assert out.read_text() == SQL+++def test_run_from_gzip_child_exiting_early_reports_its_exit(tmp_path):+ src = tmp_path / 'db.sql.gz'+ with gzip.open(src, 'wt') as f:+ f.write(SQL * 5)+ result = run_from_gzip(+ [sys.executable, '-c', 'import sys; sys.stderr.write("syntax error"); sys.exit(1)'],+ str(src))+ assert not result['success']+ assert result['returncode'] == 1+++def fake_clients(tmp_path, monkeypatch):+ """Route mysqldump/mysql to Python stand-ins; mysql records its stdin."""+ received = tmp_path / 'mysql_stdin.sql'+ stand_ins = {+ 'mysqldump': WRITE_SQL,+ 'mysql': f'import sys; open({str(received)!r}, "wb").write(sys.stdin.buffer.read())',+ }+ real_popen = subprocess.Popen++ def popen(cmd, *args, **kwargs):+ if cmd[0] in stand_ins:+ cmd = [sys.executable, '-c', stand_ins[cmd[0]]]+ return real_popen(cmd, *args, **kwargs)++ monkeypatch.setattr(system.subprocess, 'Popen', popen)+ monkeypatch.setattr('app.services.backup_service.is_command_available', lambda c: True)+ return received+++def test_mysql_backup_then_restore_round_trips(app, tmp_path, fake_clients):+ from app.services.backup_service import BackupService+ backup = BackupService._backup_database_internal(+ 'mysql', 'shop', str(tmp_path), {'user': 'root', 'password': 'pw'})+ assert backup['success'], backup+ with gzip.open(backup['path'], 'rt') as f:+ assert f.read() == SQL++ restored = BackupService.restore_database(backup['path'], 'mysql', 'shop',+ user='root', password='pw')+ assert restored['success'], restored+ assert fake_clients.read_text() == SQL