Skip to content

Beautiful Open-Source Form Builder

18.7 KB
Raw Blame
#!/usr/bin/env python3
"""Private promotion for the five-role Coolify Compose service.

The default is a read-only plan. The installed main-release controller applies
successful main releases with --apply --allow-service-restart.
"""
import argparse
import base64
import fcntl
import hashlib
import ipaddress
import inspect
import json
import os
from pathlib import Path
import re
import subprocess
import time
import urllib.request

ROLES = ('api', 'frontend', 'jobs', 'scheduler', 'migrate')


def native_image_name(image):
    """Docker archives retain tags, not registry RepoDigests, across engines."""
    package, digest = image.split('@sha256:')
    role = package.rsplit('/', 1)[-1]
    return 'opnform-' + role + ':sha256-' + digest


def image_fingerprint(meta):
    # Image IDs can denote a config or an OCI index depending on the engine.
    # Bind the actual layers and runtime configuration instead of comparing types.
    record = {name: meta[name] for name in ('Architecture', 'Os', 'RootFS')}
    # Classic Docker adds legacy container-creation fields that containerd omits.
    # Missing and null OCI defaults have the same runtime meaning.
    record['Config'] = {name: meta['Config'].get(name) for name in (
        'Env', 'Entrypoint', 'Cmd', 'User', 'WorkingDir', 'Labels', 'ExposedPorts',
        'Volumes', 'StopSignal', 'Healthcheck', 'Shell')}
    return hashlib.sha256(json.dumps(record, sort_keys=True).encode()).hexdigest()


def execute(args, timeout=120, **kwargs):
    result = subprocess.run(args, capture_output=True, timeout=timeout, **kwargs)
    if result.returncode:
        raise RuntimeError(args[0] + ' failed; remote output withheld')
    return result.stdout


def validate_release(release):
    if (release.get('schema') != 'opnform-coolify-service-release/v1'
            or release.get('repository') != 'OpnForm/OpnForm'
            or release.get('ref') != 'refs/heads/main'
            or not re.fullmatch('[0-9a-f]{40}', release.get('sha', ''))
            or type(release.get('automatic_deployment')) is not bool):
        raise ValueError('A validated OpnForm main release is required')
    for name in ('run_id', 'run_attempt'):
        if type(release.get(name)) is not int or release[name] <= 0:
            raise ValueError('Missing CI run identity')
    if set(release.get('images', {})) != {'backend', 'frontend'}:
        raise ValueError('A complete image pair is required')
    for role, image in release['images'].items():
        if not re.fullmatch(re.escape('ghcr.io/opnform/hosted-' + role + '@sha256:') + '[0-9a-f]{64}', image):
            raise ValueError('Refusing a foreign or mutable image reference')


def validate_target(target):
    if (target.get('product') != 'opnform'
            or not re.fullmatch('[a-z0-9]{20,32}', target.get('service_uuid', ''))):
        raise ValueError('Explicit private OpnForm service allowlist required')
    for name in ('server_id', 'project_id', 'team_id'):
        if type(target.get(name)) is not int or target[name] < 0:
            raise ValueError('Invalid private resource allowlist')
    for name in ('controller_alias', 'native_alias'):
        if not re.fullmatch('[a-zA-Z0-9_-]+', target.get(name, '')):
            raise ValueError('Invalid SSH alias')
    if ipaddress.ip_address(target['native_address']) not in ipaddress.ip_network('100.64.0.0/10'):
        raise ValueError('The allowlisted native server must use Tailscale')
    config = Path(target['ssh_config'])
    if not config.is_absolute() or not config.is_file() or config.stat().st_mode & 0o077:
        raise ValueError('Use a private absolute SSH config file')
    for role in ('backend', 'frontend'):
        if not re.fullmatch('sha256:[0-9a-f]{64}', target.get('expected_current_images', {}).get(role, '')):
            raise ValueError('Pin the currently running image identities before promotion')
    for name in ('api_url', 'frontend_url'):
        expected = 'https://api.opnform.com' if name == 'api_url' else 'https://opnform.com'
        if target.get(name) != expected:
            raise ValueError('Refusing another product domain')


def ssh(target, alias):
    base = ['ssh', '-F', target['ssh_config'], '-o', 'BatchMode=yes', '-o',
            'StrictHostKeyChecking=yes', '-o', 'UpdateHostKeys=no', '-o',
            'ForwardAgent=no', '-o', 'IdentitiesOnly=yes', alias]
    effective = execute(base[:-1] + ['-G', alias], text=True)
    options = dict(line.split(' ', 1) for line in effective.splitlines() if ' ' in line)
    try:
        address = ipaddress.ip_address(options['hostname'])
        if address not in ipaddress.ip_network('100.64.0.0/10'):
            raise ValueError('SSH must remain on Tailscale')
        if alias == target['native_alias'] and str(address) != target['native_address']:
            raise ValueError('Native SSH alias differs from the allowlisted server address')
    except (KeyError, ValueError):
        raise ValueError('SSH must use an explicitly pinned Tailscale address') from None
    return base


def verify_ci(release):
    run = github('/repos/OpnForm/OpnForm/actions/runs/' + str(release['run_id']))
    if (run.get('name') != 'CI/CD Pipeline' or run.get('head_sha') != release['sha']
            or run.get('head_branch') != 'main' or run.get('event') != 'push'
            or run.get('status') != 'completed' or run.get('conclusion') != 'success'
            or run.get('run_attempt') != release['run_attempt']):
        raise ValueError('The exact main revision needs a fully successful CI run')
    current = github('/repos/OpnForm/OpnForm/commits/main')
    if current['sha'] != release['sha']:
        raise ValueError('A newer main revision exists; do not silently deploy stale code')


def github(path):
    token = os.environ.get('GH_TOKEN')
    if not token:
        return json.loads(execute(['gh', 'api', path], text=True))
    request = urllib.request.Request('https://api.github.com' + path, headers={
        'Authorization': 'Bearer ' + token, 'Accept': 'application/vnd.github+json',
        'User-Agent': 'opnform-production-release'})
    with urllib.request.urlopen(request, timeout=30) as response:
        return json.load(response)


def controller(target, release, apply=False, finish=False):
    payload = base64.b64encode(json.dumps({'target': target, 'release': release,
        'release_id': release['sha'] + '-' + str(release['run_id']) + '-' + str(release['run_attempt']),
        'native_images': {role: native_image_name(image) for role, image in release['images'].items()},
        'apply': apply, 'finish': finish}).encode()).decode()
    program = r'''<?php
require '/var/www/html/vendor/autoload.php';
$app=require '/var/www/html/bootstrap/app.php';
$app->make(Illuminate\Contracts\Console\Kernel::class)->bootstrap();
$x=json_decode(base64_decode('__PAYLOAD__'),true,512,JSON_THROW_ON_ERROR);$t=$x['target'];
$s=App\Models\Service::where('uuid',$t['service_uuid'])->sole();
if($s->server_id!==$t['server_id']||$s->environment->project->id!==$t['project_id']||$s->environment->project->team_id!==$t['team_id'])throw new RuntimeException('Resource allowlist mismatch');
if($s->server->ip!==$t['native_address'])throw new RuntimeException('Native address mismatch');
$raw=json_decode($s->docker_compose_raw,true,512,JSON_THROW_ON_ERROR);
$roles=['api','frontend','jobs','scheduler','migrate'];
if(array_diff($roles,array_keys($raw['services']))||array_diff(array_keys($raw['services']),$roles))throw new RuntimeException('Unexpected service roles');
$dir='/var/www/html/storage/app/opnform-release-history/'.$s->uuid;$lease=$dir.'/.promotion-active';
if($x['finish']){
 $active=json_decode(file_get_contents($lease),true,512,JSON_THROW_ON_ERROR);
 if($active['release_id']!==$x['release_id'])throw new RuntimeException('Promotion lease differs');
 if(!unlink($lease))throw new RuntimeException('Cannot release promotion lease');
}elseif(file_exists($lease))throw new RuntimeException('Prior promotion needs inspection');
if($x['apply']){
 if(!is_dir($dir)&&!mkdir($dir,0700,true))throw new RuntimeException('Cannot save previous configuration');
 $handle=fopen($lease,'x');if($handle===false)throw new RuntimeException('Another promotion acquired the service');
 chmod($lease,0600);fwrite($handle,json_encode(['release_id'=>$x['release_id'],'started_at'=>time()]));fclose($handle);
 $file=$dir.'/'.$x['release_id'].'.json';
 if(file_exists($file))throw new RuntimeException('Prior promotion needs inspection');
 if(file_put_contents($file,$s->docker_compose_raw,LOCK_EX)===false)throw new RuntimeException('Cannot save previous configuration');chmod($file,0600);
 foreach($roles as$r)$raw['services'][$r]['image']=$x['native_images'][$r==='frontend'?'frontend':'backend'];
 $frontend=&$raw['services']['frontend'];
 $frontend['mem_limit']='2g';
 if(isset($frontend['deploy']['resources']['limits']['memory']))$frontend['deploy']['resources']['limits']['memory']='2g';
 $env=$frontend['environment']??[];
 $nodeOptions='';
 if(array_is_list($env)){
  foreach($env as$v)if(str_starts_with($v,'NODE_OPTIONS='))$nodeOptions=substr($v,13);
  $env=array_values(array_filter($env,fn($v)=>!str_starts_with($v,'NODE_OPTIONS=')));
 }else{$nodeOptions=$env['NODE_OPTIONS']??'';}
 $nodeOptions=trim(preg_replace('/(?:^|\\s)--max-old-space-size(?:=|\\s+)\\d+(?=\\s|$)/','',$nodeOptions)).' --max-old-space-size=1536';
 if(array_is_list($env)){
  $env[]='NODE_OPTIONS='.trim($nodeOptions);
 }else{$env['NODE_OPTIONS']=trim($nodeOptions);}
 $frontend['environment']=$env;
 unset($frontend);

 $s->docker_compose_raw=json_encode($raw,JSON_THROW_ON_ERROR);$s->save();$s->parse();$s->saveComposeConfigs();
}
echo json_encode(['service_uuid'=>$s->uuid,'server_id'=>$s->server_id,'images_selected'=>$x['apply'],'containers_restarted'=>false]);
'''.replace('__PAYLOAD__', payload)
    command = (['docker', 'exec', '-i', 'coolify', 'php'] if target.get('controller_local') is True
               else ssh(target, target['controller_alias']) + ['docker exec -i coolify php'])
    return json.loads(execute(command, input=program, text=True))


def native(target, program):
    return json.loads(execute(ssh(target, target['native_alias']) + ['python3 -'], input=program, text=True))


def check_current(target):
    payload = json.dumps({'uuid': target['service_uuid'], 'expected': target['expected_current_images']})
    return native(target, """import subprocess,json
x=json.loads(__PAYLOAD__);roles=['api','frontend','jobs','scheduler']
rows=json.loads(subprocess.check_output(['docker','inspect',*[r+'-'+x['uuid']for r in roles]]))
for r in rows:
 role=r['Name'].lstrip('/').split('-')[0];component='frontend'if role=='frontend'else'backend'
 assert r['Image']==x['expected'][component],'Live image identity drifted'
 assert r['State']['Running']and r['State'].get('Health',{}).get('Status')=='healthy','Live role is not healthy'
print(json.dumps({'current_roles_healthy':True,'current_image_identities_match':True}))
""".replace('__PAYLOAD__', repr(payload)))


def stage_images(target, release):
    for image in release['images'].values():
        execute(['docker', 'pull', '--platform', 'linux/amd64', image], timeout=900)
        meta = json.loads(execute(['docker', 'image', 'inspect', image], text=True))[0]
        if (meta['Config']['Labels'].get('org.opencontainers.image.revision') != release['sha']
                or meta['Architecture'] != 'amd64' or meta['Os'] != 'linux'):
            raise ValueError('Image revision differs from the reviewed release')
        selected = native_image_name(image)
        execute(['docker', 'tag', image, selected])
        source = subprocess.Popen(['docker', 'save', selected], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
        try:
            loaded = subprocess.run(ssh(target, target['native_alias']) + ['docker load'],
                                    stdin=source.stdout, capture_output=True, timeout=1200)
            source.stdout.close()
            if source.wait(timeout=30) or loaded.returncode:
                raise RuntimeError('Private image staging failed; live containers unchanged')
        finally:
            if source.poll() is None:
                source.terminate()
        expected = image_fingerprint(meta)
        observed = native(target, 'import subprocess,json,hashlib\n' + inspect.getsource(image_fingerprint)
            + "\nx=json.loads(subprocess.check_output(['docker','image','inspect'," + repr(selected)
            + "]))[0]\nprint(json.dumps({'fingerprint':image_fingerprint(x)}))\n")
        if observed['fingerprint'] != expected:
            raise ValueError('Staged image layers/configuration differ from the registry image')


def remember_current_images(target, release, path):
    """Advance the private image pins only after every role and public gate passed."""
    references = {role: native_image_name(image) for role, image in release['images'].items()}
    payload = repr(json.dumps({'uuid': target['service_uuid'], 'images': references}))
    images = native(target, """import json,subprocess
x=json.loads(__PAYLOAD__)
expected={role:json.loads(subprocess.check_output(['docker','image','inspect',image]))[0]['Id']for role,image in x['images'].items()}
for role in ['api','frontend','jobs','scheduler']:
 row=json.loads(subprocess.check_output(['docker','inspect',role+'-'+x['uuid']]))[0]
 assert row['State']['Running']and row['State'].get('Health',{}).get('Status')=='healthy'
 assert row['Image']==expected['frontend'if role=='frontend'else'backend']
print(json.dumps(expected))
""".replace('__PAYLOAD__', payload))
    updated = {**target, 'expected_current_images': images}
    temporary = path.with_suffix('.next')
    temporary.write_text(json.dumps(updated, indent=2) + '\n')
    temporary.chmod(0o600)
    temporary.replace(path)


def promote(target, release, journal):
    payload = repr(json.dumps({'uuid': target['service_uuid'], 'sha': release['sha']}))
    program = r'''import subprocess,json,pathlib,time
x=json.loads(__PAYLOAD__);base=pathlib.Path('/data/coolify/services')/x['uuid'];assert base.is_dir()
compose=['docker','compose','--project-directory',str(base),'--env-file',str(base/'.env'),'-f',str(base/'docker-compose.yml')]
def run(args):
 p=subprocess.run(args,capture_output=True,text=True,timeout=4000)
 if p.returncode:raise RuntimeError('Role operation failed; inspect private Docker logs')
 return p.stdout.strip()
def healthy(name,seconds=240):
 end=time.monotonic()+seconds
 while time.monotonic()<end:
  row=json.loads(run(['docker','inspect',name]))[0]
  if row['State'].get('Health',{}).get('Status')=='healthy':return
  if not row['State']['Running']:raise RuntimeError('Role exited before health')
  time.sleep(3)
 raise RuntimeError('Health timeout; inspect before retry')
migration='opnform-release-migrate-'+x['sha'][:16]
run(compose+['--profile','schema-migration','run','-d','--no-deps','--pull','never','--name',migration,'migrate'])
healthy(migration,1800)
# Migration readiness is a marker; its process intentionally keeps sleeping.
run(['docker','stop','--time','20',migration]);run(['docker','rm',migration])
for role in ['jobs','scheduler']:
 run(['docker','stop','--time','3700',role+'-'+x['uuid']])
for role,profile in [('api','application-prepared'),('frontend','application-prepared'),('jobs','writers-approved'),('scheduler','writers-approved')]:
 run(compose+['--profile',profile,'up','-d','--no-deps','--pull','never',role]);healthy(role+'-'+x['uuid'])
print(json.dumps({'roles_healthy':True,'revision':x['sha']}))
'''.replace('__PAYLOAD__', payload)
    # The long wait is confined to an explicitly requested promotion, never CI/merge.
    result = subprocess.run(ssh(target, target['native_alias']) + ['python3 -'], input=program,
                            text=True, capture_output=True, timeout=12000)
    if result.returncode:
        raise RuntimeError('Promotion stopped; inspect journal and private role logs before retry')
    journal['runtime'] = json.loads(result.stdout)


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('release', type=Path)
    parser.add_argument('--target', required=True, type=Path)
    parser.add_argument('--apply', action='store_true')
    parser.add_argument('--allow-service-restart', action='store_true')
    parser.add_argument('--journal', type=Path)
    args = parser.parse_args()
    release = json.loads(args.release.read_text()); target = json.loads(args.target.read_text())
    validate_release(release); validate_target(target); verify_ci(release)
    controller(target, release); check_current(target)
    if not args.apply:
        print(json.dumps({'plan_only': True, 'sha': release['sha'], 'service': target['service_uuid'],
                          'images': release['images'], 'roles': ROLES, 'live_containers_unchanged': True}))
        return
    if not args.allow_service_restart or not args.journal:
        parser.error('Promotion requires --allow-service-restart and a private --journal')
    os.umask(0o077)
    with args.journal.with_suffix('.lock').open('w') as lock:
        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
        if args.journal.exists():
            raise RuntimeError('A prior promotion journal exists; reconcile it before retry')
        stage_images(target, release)
        verify_ci(release)
        check_current(target)
        journal = {'sha': release['sha'], 'started_at': time.time(), 'status': 'selecting-images',
                   'previous_images': target['expected_current_images']}
        def save():
            args.journal.write_text(json.dumps(journal, indent=2) + '\n')
        save()
        try:
            controller(target, release, apply=True)
            journal['status'] = 'migrating-and-promoting'; save()
            promote(target, release, journal)
            for base, route in [(target['api_url'], '/healthz'), (target['frontend_url'], '/release.json')]:
                with urllib.request.urlopen(base + route, timeout=20) as response:
                    if json.load(response).get('revision') != release['sha']:
                        raise RuntimeError('Public revision does not match the selected release')
            remember_current_images(target, release, args.target)
            controller(target, release, finish=True)
            journal['status'] = 'success'; journal['finished_at'] = time.time(); save()
        except Exception:
            journal['status'] = 'needs-inspection'; save()
            raise
        if target.get('controller_local') is True:
            # Only this controller's just-staged references; never native images
            # or a shared Docker prune. Images still in use cannot be removed.
            for image in release['images'].values():
                for reference in (native_image_name(image), image):
                    subprocess.run(['docker', 'image', 'rm', reference], capture_output=True, timeout=60)
        print('Selected release is healthy. AWS, DNS, databases and other services were not reconfigured.')


if __name__ == '__main__':
    main()