Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp
20.0 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-or-later

use crate::desktop::{copy_tree, remove_path};
use crate::paths::{DEV_STATE_DIR, ROOT};
use crate::proc::{RunOptions, run_command};
use anyhow::{Context, Result, bail, ensure};
use serde_json::Value;
use sha2::{Digest, Sha256};
use std::env;
use std::fs;
use std::io;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::{SystemTime, UNIX_EPOCH};

pub const DEVELOPMENT_PACKAGE_ORIGIN: &str = "http://127.0.0.1:48780";
const PKGS_ROOT_ENV: &str = "FLUXER_DEV_PKGS_ROOT";
const PKGS_CONTAINER: &str = "fluxer-dev-pkgs";
const PKGS_IMAGE: &str = "nginx:1.28";
const PKGS_PUBLISH: &str = "127.0.0.1:48780:80";
const PKGS_CONTAINER_ROOT: &str = "/srv/pkgs";
const PKGS_CONTAINER_CONFIG: &str = "/etc/nginx/conf.d/default.conf";
const PKGS_NGINX_CONFIG: &str = "server {
    listen 80;
    root /srv/pkgs;
    etag on;
    location ~ /modules\\.json$ {
        add_header Cache-Control \"max-age=0, must-revalidate\" always;
    }
    location ~ /package\\.br$ {
        add_header Cache-Control \"public, max-age=31536000, immutable\" always;
    }
}
";
const MODULES_KEY_SEGMENT: &str = "modules";
const MODULE_PACKAGE_NAME: &str = "package.br";
const CHANNEL_MANIFEST_NAME: &str = "modules.json";

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ModuleCoordinate {
    pub channel: String,
    pub platform: String,
    pub arch: String,
}

#[derive(Debug, Clone, PartialEq, Eq)]
struct PackedModulesStamp {
    fingerprint: String,
    version: String,
    pub_date: String,
}

impl PackedModulesStamp {
    fn parse(raw: &str) -> Option<Self> {
        let mut parts = raw.split_whitespace();
        let stamp = Self {
            fingerprint: parts.next()?.to_owned(),
            version: parts.next()?.to_owned(),
            pub_date: parts.next()?.to_owned(),
        };
        parts.next().is_none().then_some(stamp)
    }

    fn render(&self) -> String {
        format!("{} {} {}\n", self.fingerprint, self.version, self.pub_date)
    }
}

fn modules_state_dir() -> PathBuf {
    DEV_STATE_DIR.join("desktop-modules")
}

fn modules_workdir() -> PathBuf {
    modules_state_dir().join("work")
}

fn modules_stamp_path() -> PathBuf {
    modules_state_dir().join("modules.stamp")
}

fn publish_staging_dir() -> PathBuf {
    modules_state_dir().join("publish")
}

fn pkgs_state_dir() -> PathBuf {
    DEV_STATE_DIR.join("desktop-pkgs")
}

pub fn pkgs_root() -> PathBuf {
    match env::var(PKGS_ROOT_ENV) {
        Ok(value) if !value.trim().is_empty() => PathBuf::from(value.trim()),
        _ => pkgs_state_dir().join("root"),
    }
}

pub fn host_module_platform() -> Result<&'static str> {
    match env::consts::OS {
        "macos" => Ok("darwin"),
        "linux" => Ok("linux"),
        "windows" => Ok("win32"),
        other => bail!("no desktop module platform for host OS {other}"),
    }
}

pub fn rfc3339_utc(now: SystemTime) -> String {
    chrono::DateTime::<chrono::Utc>::from(now)
        .format("%Y-%m-%dT%H:%M:%SZ")
        .to_string()
}

fn run_ci_step(step: &str, cwd: &Path, env: Vec<(String, Option<String>)>) -> Result<()> {
    let manifest = ROOT.join("tools/ci/Cargo.toml");
    let manifest = manifest.display().to_string();
    run_command(
        &[
            "cargo",
            "run",
            "--quiet",
            "--manifest-path",
            &manifest,
            "--",
            "build-desktop",
            "--step",
            step,
        ],
        RunOptions {
            cwd,
            env,
            load_default_env: false,
            ..RunOptions::default()
        },
    )
    .map(drop)
}

fn source_sha() -> String {
    Command::new("git")
        .args(["rev-parse", "HEAD"])
        .current_dir(ROOT.as_path())
        .stderr(Stdio::null())
        .output()
        .ok()
        .filter(|output| output.status.success())
        .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_owned())
        .unwrap_or_default()
}

fn pack_renderer_modules(
    renderer: &Path,
    fingerprint: &str,
    version: &str,
    channel: &str,
) -> Result<PackedModulesStamp> {
    let workdir = modules_workdir();
    let stamp_path = modules_stamp_path();
    if let Some(stamp) = fs::read_to_string(&stamp_path)
        .ok()
        .as_deref()
        .and_then(PackedModulesStamp::parse)
        && stamp.fingerprint == fingerprint
        && workdir.join("desktop-modules").is_dir()
    {
        println!(
            "Reusing the renderer modules packed at {} from unchanged sources",
            stamp.version
        );
        return Ok(stamp);
    }
    println!("Packing the renderer into desktop modules...");
    remove_path(&stamp_path)?;
    remove_path(&workdir)?;
    let renderer_copy = workdir.join("fluxer_desktop/dist/renderer");
    copy_tree(renderer, &renderer_copy, &|_| false)?;
    let static_link = workdir.join("fluxer_static");
    #[cfg(unix)]
    std::os::unix::fs::symlink(ROOT.join("fluxer_static"), &static_link)
        .with_context(|| format!("failed to link {}", static_link.display()))?;
    #[cfg(not(unix))]
    copy_tree(&ROOT.join("fluxer_static"), &static_link, &|_| false)?;
    let env = vec![
        ("WORKDIR".to_owned(), Some(workdir.display().to_string())),
        ("BUILD_VERSION".to_owned(), Some(version.to_owned())),
        ("BUILD_CHANNEL".to_owned(), Some(channel.to_owned())),
        ("SOURCE_SHA".to_owned(), Some(source_sha())),
    ];
    run_ci_step("split_modules", &workdir, env.clone())?;
    run_ci_step("pack_modules", &workdir, env)?;
    remove_path(&workdir.join("fluxer_desktop"))?;
    let stamp = PackedModulesStamp {
        fingerprint: fingerprint.to_owned(),
        version: version.to_owned(),
        pub_date: rfc3339_utc(SystemTime::now()),
    };
    fs::write(&stamp_path, stamp.render())
        .with_context(|| format!("failed to write {}", stamp_path.display()))?;
    Ok(stamp)
}

fn stage_channel_manifest(
    stamp: &PackedModulesStamp,
    target: &ModuleCoordinate,
) -> Result<PathBuf> {
    let staging = publish_staging_dir();
    remove_path(&staging)?;
    let payload_root = staging.join("payload_tree/desktop");
    fs::create_dir_all(&payload_root)
        .with_context(|| format!("failed to create {}", payload_root.display()))?;
    run_ci_step(
        "build_module_manifest",
        &staging,
        vec![
            (
                "WORKDIR".to_owned(),
                Some(modules_workdir().display().to_string()),
            ),
            ("BUILD_CHANNEL".to_owned(), Some(target.channel.clone())),
            ("BUILD_VERSION".to_owned(), Some(stamp.version.clone())),
            ("PUB_DATE".to_owned(), Some(stamp.pub_date.clone())),
            ("DESKTOP_MODULE_ONLY".to_owned(), Some("1".to_owned())),
            ("PLATFORM".to_owned(), Some(target.platform.clone())),
            ("ARCH".to_owned(), Some(target.arch.clone())),
            (
                "PKGS_DOWNLOAD_BASE_URL".to_owned(),
                Some(DEVELOPMENT_PACKAGE_ORIGIN.to_owned()),
            ),
        ],
    )?;
    Ok(payload_root)
}

fn sha256_file(path: &Path) -> Result<String> {
    let mut file =
        fs::File::open(path).with_context(|| format!("failed to open {}", path.display()))?;
    let mut hasher = Sha256::new();
    let mut buffer = vec![0u8; 1 << 20];
    loop {
        let read = io::Read::read(&mut file, &mut buffer)
            .with_context(|| format!("failed to read {}", path.display()))?;
        if read == 0 {
            break;
        }
        hasher.update(&buffer[..read]);
    }
    Ok(hex::encode(hasher.finalize()))
}

fn write_atomically(path: &Path, bytes: &[u8]) -> Result<()> {
    let parent = path
        .parent()
        .with_context(|| format!("{} has no parent directory", path.display()))?;
    fs::create_dir_all(parent).with_context(|| format!("failed to create {}", parent.display()))?;
    let next = parent.join(format!(
        ".{}.next",
        path.file_name()
            .and_then(|name| name.to_str())
            .unwrap_or("file")
    ));
    fs::write(&next, bytes).with_context(|| format!("failed to write {}", next.display()))?;
    fs::rename(&next, path).with_context(|| format!("failed to move {} into place", path.display()))
}

fn copy_package(source: &Path, destination: &Path) -> Result<()> {
    if destination.is_file() {
        return Ok(());
    }
    let parent = destination
        .parent()
        .with_context(|| format!("{} has no parent directory", destination.display()))?;
    fs::create_dir_all(parent).with_context(|| format!("failed to create {}", parent.display()))?;
    let next = parent.join(".package.br.next");
    fs::copy(source, &next)
        .with_context(|| format!("failed to copy {} to {}", source.display(), next.display()))?;
    fs::rename(&next, destination)
        .with_context(|| format!("failed to move {} into place", destination.display()))
}

fn manifest_without_metadata_version(manifest: &Value) -> Value {
    let mut stripped = manifest.clone();
    if let Some(object) = stripped.as_object_mut() {
        object.remove("metadata_version");
    }
    stripped
}

fn metadata_version_of(manifest: &Value, path: &Path) -> Result<u64> {
    manifest
        .get("metadata_version")
        .and_then(Value::as_u64)
        .with_context(|| format!("{} declares no valid metadata_version", path.display()))
}

pub fn next_metadata_version(now: u64, own: u64, live: Option<u64>) -> u64 {
    let stamp = now.max(own);
    match live {
        Some(live) if stamp <= live => live + 1,
        _ => stamp,
    }
}

fn verify_manifest_packages(
    root: &Path,
    channel: &str,
    manifest: &Value,
    path: &Path,
) -> Result<()> {
    let modules = manifest
        .get("modules")
        .and_then(Value::as_object)
        .with_context(|| format!("{} lists no modules", path.display()))?;
    ensure!(
        modules.contains_key("fluxer_renderer"),
        "{} lists no fluxer_renderer",
        path.display()
    );
    for (module, entry) in modules {
        let sha256 = entry
            .get("sha256")
            .and_then(Value::as_str)
            .with_context(|| format!("{} declares no sha256 for {module}", path.display()))?;
        let bytes = entry
            .get("bytes")
            .and_then(Value::as_u64)
            .with_context(|| format!("{} declares no size for {module}", path.display()))?;
        let url = entry
            .get("url")
            .and_then(Value::as_str)
            .with_context(|| format!("{} declares no url for {module}", path.display()))?;
        let address = format!(
            "/desktop/{channel}/{MODULES_KEY_SEGMENT}/{module}/{sha256}/{MODULE_PACKAGE_NAME}"
        );
        ensure!(
            url == format!("{DEVELOPMENT_PACKAGE_ORIGIN}{address}"),
            "{} points {module} at {url}, which is not its content address",
            path.display()
        );
        let package = root.join(address.trim_start_matches('/'));
        let size = fs::metadata(&package)
            .with_context(|| {
                format!(
                    "{} is missing, {} references it",
                    package.display(),
                    path.display()
                )
            })?
            .len();
        ensure!(
            size == bytes,
            "{} is {size} bytes, {} declares {bytes}",
            package.display(),
            path.display()
        );
        let actual = sha256_file(&package)?;
        ensure!(
            actual == sha256,
            "{} hashes to {actual}, {} declares {sha256}",
            package.display(),
            path.display()
        );
    }
    Ok(())
}

fn publish_modules(
    payload_root: &Path,
    root: &Path,
    stamp: &PackedModulesStamp,
    target: &ModuleCoordinate,
) -> Result<u64> {
    let staged_modules = payload_root.join(&target.channel).join(MODULES_KEY_SEGMENT);
    for module_dir in fs::read_dir(&staged_modules)
        .with_context(|| format!("failed to read {}", staged_modules.display()))?
    {
        let module_dir = module_dir?.path();
        for digest_dir in fs::read_dir(&module_dir)
            .with_context(|| format!("failed to read {}", module_dir.display()))?
        {
            let digest_dir = digest_dir?.path();
            let relative = digest_dir.strip_prefix(payload_root)?;
            copy_package(
                &digest_dir.join(MODULE_PACKAGE_NAME),
                &root
                    .join("desktop")
                    .join(relative)
                    .join(MODULE_PACKAGE_NAME),
            )?;
        }
    }
    let coordinate_dir = root
        .join("desktop")
        .join(&target.channel)
        .join(&target.platform)
        .join(&target.arch);
    let staged_manifest_path = payload_root
        .join(&target.channel)
        .join(&target.platform)
        .join(&target.arch)
        .join(CHANNEL_MANIFEST_NAME);
    let staged_bytes = fs::read(&staged_manifest_path)
        .with_context(|| format!("failed to read {}", staged_manifest_path.display()))?;
    let version_manifest_path = coordinate_dir
        .join(&stamp.version)
        .join(CHANNEL_MANIFEST_NAME);
    write_atomically(&version_manifest_path, &staged_bytes)?;
    let manifest: Value = serde_json::from_slice(&staged_bytes)
        .with_context(|| format!("failed to parse {}", staged_manifest_path.display()))?;
    verify_manifest_packages(root, &target.channel, &manifest, &version_manifest_path)?;
    let own = metadata_version_of(&manifest, &version_manifest_path)?;
    let live_path = coordinate_dir.join(CHANNEL_MANIFEST_NAME);
    let live = match fs::read(&live_path) {
        Ok(bytes) => Some(
            serde_json::from_slice::<Value>(&bytes)
                .with_context(|| format!("failed to parse {}", live_path.display()))?,
        ),
        Err(error) if error.kind() == io::ErrorKind::NotFound => None,
        Err(error) => {
            return Err(error).with_context(|| format!("failed to read {}", live_path.display()));
        }
    };
    let live_version = live
        .as_ref()
        .map(|live| metadata_version_of(live, &live_path))
        .transpose()?;
    if let (Some(live), Some(live_version)) = (&live, live_version)
        && live_version >= own
        && manifest_without_metadata_version(live) == manifest_without_metadata_version(&manifest)
    {
        println!(
            "The local package server already serves modules {} at metadata_version {live_version}",
            stamp.version
        );
        return Ok(live_version);
    }
    let now = SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .map(|elapsed| elapsed.as_secs())
        .unwrap_or(0);
    let next = next_metadata_version(now, own, live_version);
    let mut restamped = manifest;
    restamped["metadata_version"] = Value::from(next);
    let mut bytes = serde_json::to_vec(&restamped)?;
    bytes.push(b'\n');
    write_atomically(&live_path, &bytes)?;
    println!(
        "Published modules {} to {} at metadata_version {next}",
        stamp.version,
        live_path.display()
    );
    Ok(next)
}

fn docker_output(args: &[&str]) -> Result<std::process::Output> {
    Command::new("docker")
        .args(args)
        .stdin(Stdio::null())
        .output()
        .with_context(|| format!("failed to run docker {}", args.join(" ")))
}

fn container_matches(root: &Path, config: &Path) -> Result<bool> {
    let output = docker_output(&[
        "inspect",
        "--format",
        "{{.State.Running}} {{range .Mounts}}{{.Source}}={{.Destination}} {{end}}",
        PKGS_CONTAINER,
    ])?;
    if !output.status.success() {
        return Ok(false);
    }
    let described = String::from_utf8_lossy(&output.stdout);
    let mut fields = described.split_whitespace();
    if fields.next() != Some("true") {
        return Ok(false);
    }
    let mounts: Vec<&str> = fields.collect();
    Ok(
        mounts.contains(&format!("{}={PKGS_CONTAINER_ROOT}", root.display()).as_str())
            && mounts.contains(&format!("{}={PKGS_CONTAINER_CONFIG}", config.display()).as_str()),
    )
}

fn port_holder() -> String {
    docker_output(&["ps", "--filter", "publish=48780", "--format", "{{.Names}}"])
        .ok()
        .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_owned())
        .filter(|names| !names.is_empty())
        .map_or_else(
            || "another process".to_owned(),
            |names| format!("the container {names}"),
        )
}

pub fn ensure_pkgs_server() -> Result<()> {
    let root = pkgs_root();
    fs::create_dir_all(&root).with_context(|| format!("failed to create {}", root.display()))?;
    let root = root
        .canonicalize()
        .with_context(|| format!("failed to resolve {}", root.display()))?;
    let config = pkgs_state_dir().join("nginx.conf");
    if fs::read_to_string(&config).ok().as_deref() != Some(PKGS_NGINX_CONFIG) {
        write_atomically(&config, PKGS_NGINX_CONFIG.as_bytes())?;
    }
    let config = config
        .canonicalize()
        .with_context(|| format!("failed to resolve {}", config.display()))?;
    if container_matches(&root, &config)? {
        println!(
            "The local package server {PKGS_CONTAINER} serves {} on {DEVELOPMENT_PACKAGE_ORIGIN}",
            root.display()
        );
        return Ok(());
    }
    let _ = docker_output(&["rm", "-f", PKGS_CONTAINER]);
    let root_mount = format!("{}:{PKGS_CONTAINER_ROOT}:ro", root.display());
    let config_mount = format!("{}:{PKGS_CONTAINER_CONFIG}:ro", config.display());
    let output = docker_output(&[
        "run",
        "--detach",
        "--name",
        PKGS_CONTAINER,
        "--restart",
        "unless-stopped",
        "--publish",
        PKGS_PUBLISH,
        "--volume",
        &root_mount,
        "--volume",
        &config_mount,
        PKGS_IMAGE,
    ])?;
    if !output.status.success() {
        let stderr = String::from_utf8_lossy(&output.stderr);
        if stderr.contains("address already in use") || stderr.contains("port is already allocated")
        {
            let _ = docker_output(&["rm", "-f", PKGS_CONTAINER]);
            bail!(
                "{DEVELOPMENT_PACKAGE_ORIGIN} is taken by {}, stop it so the development app can reach its module server",
                port_holder()
            );
        }
        bail!(
            "failed to start the local package server: {}",
            stderr.trim()
        );
    }
    println!(
        "Started the local package server {PKGS_CONTAINER} for {} on {DEVELOPMENT_PACKAGE_ORIGIN}",
        root.display()
    );
    Ok(())
}

fn verify_served_manifest(target: &ModuleCoordinate, metadata_version: u64) -> Result<()> {
    let url = format!(
        "{DEVELOPMENT_PACKAGE_ORIGIN}/desktop/{}/{}/{}/{CHANNEL_MANIFEST_NAME}",
        target.channel, target.platform, target.arch
    );
    let mut last = String::new();
    for _ in 0..20 {
        let output = Command::new("curl")
            .args(["--silent", "--fail", "--max-time", "5", &url])
            .stdin(Stdio::null())
            .output()
            .context("failed to run curl")?;
        if output.status.success() {
            let served: Value = serde_json::from_slice(&output.stdout)
                .with_context(|| format!("{url} served invalid JSON"))?;
            if served.get("metadata_version").and_then(Value::as_u64) == Some(metadata_version) {
                return Ok(());
            }
            last = format!(
                "it serves metadata_version {}",
                served.get("metadata_version").unwrap_or(&Value::Null)
            );
        } else {
            last = String::from_utf8_lossy(&output.stderr).trim().to_owned();
        }
        std::thread::sleep(std::time::Duration::from_millis(250));
    }
    bail!("{url} does not serve metadata_version {metadata_version}: {last}")
}

pub fn publish_renderer_modules(
    renderer: &Path,
    fingerprint: &str,
    version: &str,
    target: &ModuleCoordinate,
) -> Result<()> {
    let stamp = pack_renderer_modules(renderer, fingerprint, version, &target.channel)?;
    let payload_root = stage_channel_manifest(&stamp, target)?;
    let root = pkgs_root();
    let metadata_version = publish_modules(&payload_root, &root, &stamp, target)?;
    remove_path(&publish_staging_dir())?;
    ensure_pkgs_server()?;
    verify_served_manifest(target, metadata_version)
}