Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

17.7 KB
Raw Blame
use std::path::{Path, PathBuf};

use anyhow::{anyhow, Result};
use posthog_symbol_data::{write_symbol_data, AppleDsym, ElfDebugInfo};
use symbolic::debuginfo::{Archive, FileFormat, ObjectKind};
use tracing::{info, warn};

use crate::api::symbol_sets::SymbolSetUpload;
use crate::dsym::{source_bundle, DsymFile};

pub mod upload;

const ELF_MAGIC: &[u8; 4] = b"\x7fELF";
// Thin little-endian 64/32-bit Mach-O and the big-endian fat header — the
// on-disk layouts produced by Apple toolchains and Go, and the ones the
// symbolic archive parser understands (fat64, `0xcafebabf`, is not supported
// and stays skipped).
const MACHO_MAGICS: [[u8; 4]; 3] = [
    [0xcf, 0xfa, 0xed, 0xfe],
    [0xce, 0xfa, 0xed, 0xfe],
    [0xca, 0xfe, 0xba, 0xbe],
];

/// A native debug-info file discovered on disk, parsed and validated,
/// ready to be packaged for upload.
pub struct DebugSymbolFile {
    /// The debug id used as chunk_id: derived from the GNU build id for ELF
    /// (lowercase), or the `LC_UUID` for Mach-O (uppercase, matching the SDK
    /// and `dsym upload`)
    pub debug_id: String,
    pub path: PathBuf,
    /// Raw contents: the whole file for ELF, one architecture slice for Mach-O
    /// (a fat binary yields one `DebugSymbolFile` per slice)
    data: Vec<u8>,
    /// Size of the data, used to pick the richer candidate on debug_id collisions
    size: usize,
    format: FileFormat,
}

/// The outcome of scanning a directory for native debug symbols.
#[derive(Default)]
pub struct DiscoveryReport {
    /// Validated files, deduplicated by debug id
    pub files: Vec<DebugSymbolFile>,
    /// ELF/Mach-O executables/libraries that carry no debug info (skipped)
    pub without_debug_info: Vec<PathBuf>,
    /// ELFs with debug info but no GNU build id (cannot be uploaded)
    pub missing_build_id: Vec<PathBuf>,
    /// Mach-O binaries with debug info but no `LC_UUID` (cannot be uploaded)
    pub missing_uuid: Vec<PathBuf>,
    /// Mach-O binaries whose DWARF is compressed (`__zdebug_*` sections, Go's
    /// default on macOS), which the symbolication server can't read yet
    pub compressed_dwarf: Vec<PathBuf>,
    /// Apple dSYM bundles spotted while scanning (packaged via the dsym path)
    pub dsym_bundles: Vec<PathBuf>,
    /// Split-DWARF artifacts spotted while scanning (unsupported)
    pub split_dwarf: Vec<PathBuf>,
}

impl DebugSymbolFile {
    /// Package the file for upload: a ZIP with the binary stored as `dwarf` at
    /// the root (plus an optional `__source/` bundle), wrapped in the
    /// symbol_data container matching its format (`ElfDebugInfo` for ELF,
    /// `AppleDsym` for Mach-O — the layout the server expects per chunk kind).
    pub fn into_upload(
        self,
        release_id: Option<String>,
        include_source: bool,
        source_root: Option<&Path>,
    ) -> Result<SymbolSetUpload> {
        use std::io::{Cursor, Write};

        let mut buffer = Cursor::new(Vec::new());
        {
            let mut zip = zip::ZipWriter::new(&mut buffer);
            let options = zip::write::SimpleFileOptions::default()
                .compression_method(zip::CompressionMethod::Deflated);

            zip.start_file("dwarf", options)?;
            zip.write_all(&self.data)?;

            if include_source {
                match source_bundle::extract_source_paths_from_dwarf_bytes(&self.data) {
                    Ok(all_paths) => {
                        let filtered = source_bundle::filter_source_paths_for_upload(
                            &all_paths,
                            source_root,
                            &self.path,
                            is_native_source_path,
                        );
                        if !filtered.is_empty() {
                            match source_bundle::collect_source_files(&filtered) {
                                Ok(source_files) => {
                                    source_bundle::add_source_to_zip(&mut zip, &source_files)?;
                                }
                                Err(e) => {
                                    warn!("Failed to collect source files: {} (continuing without source)", e);
                                }
                            }
                        }
                    }
                    Err(e) => {
                        warn!(
                            "Failed to extract DWARF source paths: {} (continuing without source)",
                            e
                        );
                    }
                }
            }

            zip.finish()?;
        }

        let zip_data = buffer.into_inner();
        let wrapped = if self.format == FileFormat::MachO {
            write_symbol_data(AppleDsym { data: zip_data })?
        } else {
            write_symbol_data(ElfDebugInfo { data: zip_data })?
        };

        Ok(SymbolSetUpload {
            chunk_id: self.debug_id,
            release_id,
            data: wrapped,
            content_hash: None,
        })
    }
}

/// Drop native registry/toolchain sources outside the user's project tree.
/// Go toolchain sources are already trimmed inside the shared extraction
/// (see `filter_go_toolchain_sources`).
fn is_native_source_path(path: &str) -> bool {
    const EXCLUDED: &[&str] = &[
        "/.cargo/registry/",
        "/.cargo/git/",
        "/rustc/",
        "/.rustup/toolchains/",
        "/vendor/",
    ];
    if EXCLUDED.iter().any(|sub| path.contains(sub)) {
        tracing::debug!("Filtered out (native toolchain): {}", path);
        return false;
    }
    true
}

/// Walk `directory` and classify everything that looks like native debug
/// symbols. Only ELF and Mach-O executables, shared libraries, and debug
/// companions (e.g. from `objcopy --only-keep-debug`) are considered;
/// relocatable objects and other kinds are ignored.
pub fn discover(directory: &Path) -> Result<DiscoveryReport> {
    use walkdir::WalkDir;

    let mut report = DiscoveryReport::default();
    let mut seen: std::collections::HashMap<String, usize> = std::collections::HashMap::new();

    for entry in WalkDir::new(directory).follow_links(true) {
        // A symlink loop, dangling link, or unreadable directory surfaces here
        // as an iterator error. Skip just that entry rather than aborting the
        // whole scan, which would discard every valid file already found.
        let entry = match entry {
            Ok(entry) => entry,
            Err(e) => {
                warn!("Skipping unreadable path while scanning: {e}");
                continue;
            }
        };
        let path = entry.path();

        if path.is_dir() {
            if path.extension().is_some_and(|e| e == "dSYM") {
                report.dsym_bundles.push(path.to_path_buf());
            }
            continue;
        }

        if path.extension().is_some_and(|e| e == "dwp" || e == "dwo") {
            report.split_dwarf.push(path.to_path_buf());
            continue;
        }

        if !entry.file_type().is_file() {
            continue;
        }

        // Files inside a .dSYM bundle (its inner DWARF Mach-O) belong to the
        // bundle, which is packaged wholesale via the dsym path.
        if path
            .ancestors()
            .skip(1)
            .any(|p| p.extension().is_some_and(|e| e == "dSYM"))
        {
            continue;
        }

        for file in parse_candidates(path)? {
            match file {
                Candidate::NoDebugInfo => report.without_debug_info.push(path.to_path_buf()),
                Candidate::NoBuildId => report.missing_build_id.push(path.to_path_buf()),
                Candidate::MissingUuid => report.missing_uuid.push(path.to_path_buf()),
                Candidate::CompressedDwarf => report.compressed_dwarf.push(path.to_path_buf()),
                Candidate::Valid(file) => {
                    // Dedup by debug id (e.g. a binary and its objcopy companion):
                    // keep the larger file, which carries at least as much debug data.
                    if let Some(&existing) = seen.get(&file.debug_id) {
                        let existing_file = &report.files[existing];
                        warn!(
                            "Duplicate debug id {} for {} and {}; keeping the larger file",
                            file.debug_id,
                            existing_file.path.display(),
                            file.path.display()
                        );
                        if file.size > existing_file.size {
                            report.files[existing] = file;
                        }
                    } else {
                        seen.insert(file.debug_id.clone(), report.files.len());
                        report.files.push(file);
                    }
                }
            }
        }
    }

    Ok(report)
}

enum Candidate {
    Valid(DebugSymbolFile),
    NoDebugInfo,
    NoBuildId,
    MissingUuid,
    CompressedDwarf,
}

fn has_native_magic(path: &Path) -> bool {
    use std::io::Read;

    let Ok(mut file) = std::fs::File::open(path) else {
        warn!("Could not open {} (skipping)", path.display());
        return false;
    };
    let mut magic = [0u8; 4];
    if file.read_exact(&mut magic).is_err() {
        return false;
    }
    &magic == ELF_MAGIC || MACHO_MAGICS.contains(&magic)
}

/// Parse a file into native debug-symbol candidates. Returns an empty vec for
/// anything that isn't an ELF or Mach-O executable/library/debug companion.
/// A universal (fat) Mach-O yields one candidate per architecture slice, each
/// carrying just that slice's bytes and its own `LC_UUID`.
fn parse_candidates(path: &Path) -> Result<Vec<Candidate>> {
    // Check the magic before loading the file: build trees contain large
    // unrelated artifacts that shouldn't be read into memory just to be skipped.
    if !has_native_magic(path) {
        return Ok(Vec::new());
    }

    let Ok(data) = std::fs::read(path) else {
        warn!("Could not read {} (skipping)", path.display());
        return Ok(Vec::new());
    };

    let archive = match Archive::parse(&data) {
        Ok(archive) => archive,
        Err(e) => {
            warn!("Could not parse {} (skipping): {e}", path.display());
            return Ok(Vec::new());
        }
    };

    let mut candidates = Vec::new();
    for object in archive.objects() {
        let Ok(object) = object else {
            warn!("Unparseable object in {} (skipping it)", path.display());
            continue;
        };

        let format = object.file_format();
        if format != FileFormat::Elf && format != FileFormat::MachO {
            continue;
        }

        // Relocatable objects (.o) and other kinds aren't loadable images.
        if !matches!(
            object.kind(),
            ObjectKind::Executable | ObjectKind::Library | ObjectKind::Debug
        ) {
            continue;
        }

        if !object.has_debug_info() {
            // Go's darwin linker compresses the DWARF it embeds (`__zdebug_*`
            // sections) by default, which the server can't read — surface
            // that as its own case so the guidance can name the fix.
            if format == FileFormat::MachO && contains_zdebug_section(object.data()) {
                candidates.push(Candidate::CompressedDwarf);
            } else {
                candidates.push(Candidate::NoDebugInfo);
            }
            continue;
        }

        // Without an identity the SDK can reproduce at runtime — the GNU
        // build id for ELF, the LC_UUID for Mach-O — the upload would never
        // match any crash event. For ELF specifically, symbolic synthesizes a
        // content-derived debug id when the build id is missing, so the
        // code_id check must come first.
        let debug_id = object.debug_id();
        if (format == FileFormat::Elf && object.code_id().is_none()) || debug_id.is_nil() {
            candidates.push(match format {
                FileFormat::MachO => Candidate::MissingUuid,
                _ => Candidate::NoBuildId,
            });
            continue;
        }

        // Chunk id casing is per-format and case-sensitive on the server:
        // ELF ids are lowercase, Mach-O ids are uppercase (matching the SDKs
        // and `dsym upload`).
        let debug_id = match format {
            FileFormat::MachO => debug_id.to_string().to_uppercase(),
            _ => debug_id.to_string(),
        };

        // For a fat Mach-O, upload the slice, not the whole container: the
        // server resolves one binary per chunk id.
        let object_data = object.data().to_vec();
        candidates.push(Candidate::Valid(DebugSymbolFile {
            debug_id,
            path: path.to_path_buf(),
            size: object_data.len(),
            data: object_data,
            format,
        }));
    }

    Ok(candidates)
}

/// Detect Go-style compressed DWARF in a Mach-O slice. Section names live
/// verbatim in the section headers, so a plain byte scan is sufficient for a
/// diagnostic.
fn contains_zdebug_section(data: &[u8]) -> bool {
    data.windows(b"__zdebug_".len())
        .any(|window| window == b"__zdebug_")
}

/// Render guidance for files that couldn't be uploaded; returns an error when
/// nothing usable was found at all.
pub fn report_problems(report: &DiscoveryReport, directory: &Path) -> Result<()> {
    // ELF debug files and dSYM bundles are both uploaded by `symbol-sets upload`
    // now, so the only genuinely empty case is when neither was found.
    let nothing_to_upload = report.files.is_empty() && report.dsym_bundles.is_empty();

    if !report.split_dwarf.is_empty() {
        warn!(
            "Found {} split-DWARF file(s) (.dwp/.dwo), which aren't supported yet. \
             Build with debug info embedded in the binary (the default on Linux), \
             or upload an `objcopy --only-keep-debug` companion instead.",
            report.split_dwarf.len()
        );
    }

    for path in &report.without_debug_info {
        warn!(
            "{} has no debug info (skipping). For release builds, set \
             `debug = \"line-tables-only\"` (or `true`) in [profile.release], \
             and upload before stripping.",
            path.display()
        );
    }

    for path in &report.missing_uuid {
        warn!(
            "{} has debug info but no LC_UUID, so it cannot be matched to \
             crash events (skipping). Relink without stripping the UUID load \
             command (ld adds one by default; `-no_uuid` removes it).",
            path.display()
        );
    }

    for path in &report.compressed_dwarf {
        warn!(
            "{} carries compressed DWARF (`__zdebug` sections), which can't be \
             used for symbolication yet (skipping). Go compresses DWARF by \
             default on macOS — rebuild with `-ldflags=-compressdwarf=false` \
             and upload that binary.",
            path.display()
        );
    }

    if !report.missing_build_id.is_empty() {
        let listing = report
            .missing_build_id
            .iter()
            .map(|p| format!("  {}", p.display()))
            .collect::<Vec<_>>()
            .join("\n");
        let guidance = format!(
            "The following files have debug info but no GNU build id, so they cannot be \
             matched to crash events:\n{listing}\n\
             Link with `-Wl,--build-id=sha1` (in Rust: `-C link-arg=-Wl,--build-id=sha1` \
             via RUSTFLAGS; in Go: `-ldflags=-B=gobuildid`); most C/C++/Rust \
             toolchains add this by default, Go does not."
        );
        // Only fail the run when there's nothing valid to upload at all;
        // otherwise a stray build-id-less helper binary would block every valid
        // symbol, including any dSYM bundles found alongside it.
        if nothing_to_upload {
            anyhow::bail!("{guidance}");
        }
        warn!("{guidance}");
    }

    if nothing_to_upload {
        anyhow::bail!(
            "No ELF or Mach-O files with debug info or dSYM bundles found in {}",
            directory.display()
        );
    }

    Ok(())
}

/// Package any discovered Apple `.dSYM` bundles into uploads, reusing the exact
/// same code path as `posthog-cli dsym upload` (uppercase `LC_UUID` chunk_ids,
/// `AppleDsym` container). A bundle that fails to process — most often because
/// `dwarfdump` is unavailable, since it ships with Xcode and is macOS-only — is
/// logged and skipped rather than aborting the run, so a mixed ELF + dSYM
/// directory still uploads its ELF symbols on Linux.
///
/// Uploads come back with no release id; the caller stamps one (if any) after it
/// knows there is something to upload, so a fully-skipped run leaves no release.
pub fn package_dsym_bundles(
    bundles: &[PathBuf],
    include_source: bool,
    source_root: Option<&Path>,
) -> Vec<SymbolSetUpload> {
    let mut uploads = Vec::new();
    for bundle in bundles {
        match DsymFile::new(bundle, include_source, source_root) {
            Ok(dsym_file) => {
                info!(
                    "Processing dSYM {} (UUIDs: {})",
                    bundle.display(),
                    dsym_file.uuids().join(", ")
                );
                uploads.extend(dsym_file.into_uploads());
            }
            Err(e) => {
                warn!(
                    "Failed to process dSYM {}: {e} (skipping)",
                    bundle.display()
                );
            }
        }
    }
    uploads
}

/// Extract the debug id of an ELF file, e.g. for testing parity with the SDK.
pub fn elf_debug_id(path: &Path) -> Result<String> {
    let data = std::fs::read(path)?;
    let archive = Archive::parse(&data).map_err(|e| anyhow!("parse {}: {e}", path.display()))?;
    let object = archive
        .objects()
        .next()
        .ok_or_else(|| anyhow!("no objects in {}", path.display()))?
        .map_err(|e| anyhow!("parse object in {}: {e}", path.display()))?;
    Ok(object.debug_id().to_string())
}