Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

13.3 KB
Raw Blame
//! The corpus check (ignored by default): every `.pdf` and `.ai` file under
//! `PDF_CORPUS_DIR` (third-party files, kept out of the repository) is
//! opened, its pages listed, every object resolved, every stream decoded,
//! and every page's content parsed and written back, with counts and
//! timings reported. With `PDF_CORPUS_OUT` set, each file's pages and
//! content stream hashes are written there as JSON, with the file
//! rewritten whole (`write::file`) and updated incrementally
//! (`write::incremental`), for comparison with another reader.
//!
//! `PDF_CORPUS_DIR=… cargo test -p ai_engine --release -- --ignored --nocapture corpus`

use super::*;
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};

#[derive(Default)]
struct Totals {
    files: usize,
    failed: usize,
    repaired: usize,
    pages: usize,
    objects: usize,
    streams: usize,
    images: usize,
    stream_errors: usize,
    ops: usize,
    mismatches: usize,
    time: Duration,
}

#[test]
#[ignore = "reads a local corpus: set PDF_CORPUS_DIR"]
fn corpus() {
    let Some(dir) = std::env::var_os("PDF_CORPUS_DIR") else {
        eprintln!("PDF_CORPUS_DIR is not set; skipping");
        return;
    };
    let out = std::env::var_os("PDF_CORPUS_OUT").map(PathBuf::from);
    if let Some(out) = &out {
        std::fs::create_dir_all(out).unwrap();
    }
    let dir = PathBuf::from(dir);
    let mut files = Vec::new();
    collect(&dir, &mut files);
    files.sort();
    let mut totals = Totals::default();
    // `open`: reading the cross-reference; `first`: that, the page list,
    // and page 1's content decoded and parsed; `all`: every object
    // resolved, every stream decoded (`MB out`), every page parsed and
    // written back.
    println!(
        "{:<48} {:>9} {:>7} {:>7} {:>5} {:>6} {:>7} {:>4} {:>4} {:>7} {:>7} {:>8} {:>8}",
        "file",
        "bytes",
        "open ms",
        "1st ms",
        "pages",
        "objs",
        "streams",
        "img",
        "err",
        "MB out",
        "ops",
        "all ms",
        "rewrite"
    );
    for path in &files {
        let name = path
            .strip_prefix(&dir)
            .unwrap_or(path)
            .to_string_lossy()
            .into_owned();
        let bytes: Arc<[u8]> = Arc::from(std::fs::read(path).unwrap());
        totals.files += 1;
        let size = bytes.len();
        let start = Instant::now();
        let pdf = match Pdf::open(bytes.clone()) {
            Ok(pdf) => pdf,
            Err(e) => {
                totals.failed += 1;
                println!("{:<48} {size:>9} failed: {e}", short(&name));
                continue;
            }
        };
        let open = start.elapsed();
        if let Some(page) = pdf.pages().first() {
            let streams = content_streams(&pdf, &page.dict);
            std::hint::black_box(content::parse(&joined(&streams)));
        }
        let first = start.elapsed();
        let start = Instant::now();
        let c = check(&pdf);
        let all = start.elapsed();
        totals.time += all;
        totals.repaired += usize::from(pdf.repaired());
        totals.pages += c.pages.len();
        totals.objects += c.objects;
        totals.streams += c.streams;
        totals.images += c.images;
        totals.stream_errors += c.stream_errors;
        totals.ops += c.ops;
        totals.mismatches += c.mismatches;
        let rewrite = match &out {
            Some(out) => rewrite(&pdf, &bytes, &c.pages, out, &name),
            None => "-".into(),
        };
        println!(
            "{:<48} {size:>9} {:>7.1} {:>7.1} {:>5} {:>6} {:>7} {:>4} {:>4} {:>7.1} {:>7} {:>8.1} {:>8}{}{}",
            short(&name),
            open.as_secs_f64() * 1e3,
            first.as_secs_f64() * 1e3,
            c.pages.len(),
            c.objects,
            c.streams,
            c.images,
            c.stream_errors,
            c.decoded as f64 / 1e6,
            c.ops,
            all.as_secs_f64() * 1e3,
            rewrite,
            if pdf.repaired() { " repaired" } else { "" },
            if c.mismatches > 0 {
                format!(" {} round-trip mismatches", c.mismatches)
            } else {
                String::new()
            },
        );
    }
    println!(
        "\n{} files ({} failed to open, {} repaired): {} pages, {} objects, {} streams \
         ({} image codecs, {} errors), {} operators, {} content round-trip mismatches; \
         reading took {:.0} ms",
        totals.files,
        totals.failed,
        totals.repaired,
        totals.pages,
        totals.objects,
        totals.streams,
        totals.images,
        totals.stream_errors,
        totals.ops,
        totals.mismatches,
        totals.time.as_secs_f64() * 1e3
    );
}

/// The corpus files damaged: bytes changed, removed, and inserted, and the
/// files cut short, each read through without panicking.
#[test]
#[ignore = "reads a local corpus: set PDF_CORPUS_DIR"]
fn corpus_damage() {
    let Some(dir) = std::env::var_os("PDF_CORPUS_DIR") else {
        eprintln!("PDF_CORPUS_DIR is not set; skipping");
        return;
    };
    const MUTATIONS: usize = 40;
    const MAX_SIZE: usize = 4 << 20;
    let mut files = Vec::new();
    collect(Path::new(&dir), &mut files);
    files.sort();
    let mut rng = super::damage::Rng(0x2545_f491_4f6c_dd1d);
    let start = Instant::now();
    let mut walks = 0;
    for path in &files {
        let base = std::fs::read(path).unwrap();
        if base.is_empty() || base.len() > MAX_SIZE {
            continue;
        }
        for _ in 0..MUTATIONS {
            let mut bytes = base.clone();
            for _ in 0..1 + rng.below(8) {
                let at = rng.below(bytes.len());
                match rng.below(3) {
                    0 => bytes[at] = rng.next() as u8,
                    1 => {
                        bytes.remove(at);
                    }
                    _ => bytes.insert(at, rng.next() as u8),
                }
            }
            super::damage::walk(&bytes);
            walks += 1;
        }
        for _ in 0..8 {
            super::damage::walk(&base[..rng.below(base.len())]);
            walks += 1;
        }
    }
    println!(
        "{walks} damaged files read in {:.0} ms",
        start.elapsed().as_secs_f64() * 1e3
    );
}

fn collect(dir: &Path, out: &mut Vec<PathBuf>) {
    let Ok(entries) = std::fs::read_dir(dir) else {
        return;
    };
    for entry in entries.flatten() {
        let path = entry.path();
        if path.is_dir() {
            collect(&path, out);
        } else if path
            .extension()
            .is_some_and(|e| e.eq_ignore_ascii_case("pdf") || e.eq_ignore_ascii_case("ai"))
        {
            out.push(path);
        }
    }
}

fn short(name: &str) -> String {
    let chars: Vec<char> = name.chars().collect();
    let start = chars.len().saturating_sub(48);
    chars[start..].iter().collect()
}

/// What reading a whole file found.
struct Checked {
    /// Each page's content streams: object number and decoded data.
    pages: Vec<Vec<(u32, Vec<u8>)>>,
    objects: usize,
    streams: usize,
    images: usize,
    stream_errors: usize,
    /// Bytes the streams decoded to.
    decoded: usize,
    ops: usize,
    mismatches: usize,
}

fn check(pdf: &Pdf) -> Checked {
    let mut c = Checked {
        pages: Vec::new(),
        objects: 0,
        streams: 0,
        images: 0,
        stream_errors: 0,
        decoded: 0,
        ops: 0,
        mismatches: 0,
    };
    for r in pdf.object_refs() {
        let Some(o) = pdf.get(r) else {
            continue;
        };
        c.objects += 1;
        if let Object::Stream(s) = o {
            c.streams += 1;
            match pdf.decode_stream(&s) {
                Ok((data, codec)) => {
                    c.decoded += data.len();
                    c.images += usize::from(codec.is_some());
                }
                Err(_) => c.stream_errors += 1,
            }
        }
    }
    for page in pdf.pages() {
        let streams = content_streams(pdf, &page.dict);
        let ops = content::parse(&joined(&streams));
        c.ops += ops.len();
        let again = content::parse(&content::write(&ops));
        if ops.len() != again.len() || ops.iter().zip(&again).any(|(a, b)| !same_op(a, b)) {
            c.mismatches += 1;
        }
        c.pages.push(streams);
    }
    c
}

/// Content streams joined as one (a line break between them).
fn joined(streams: &[(u32, Vec<u8>)]) -> Vec<u8> {
    streams
        .iter()
        .flat_map(|(_, d)| d.iter().copied().chain([b'\n']))
        .collect()
}

/// A page's content streams, by object number.
fn content_streams(pdf: &Pdf, page: &Dict) -> Vec<(u32, Vec<u8>)> {
    let refs: Vec<ObjRef> = match page.get("Contents") {
        Some(Object::Ref(r)) => match pdf.get(*r) {
            Some(Object::Array(a)) => a.iter().filter_map(Object::as_ref).collect(),
            Some(Object::Stream(_)) => vec![*r],
            _ => Vec::new(),
        },
        Some(Object::Array(a)) => a.iter().filter_map(Object::as_ref).collect(),
        _ => Vec::new(),
    };
    refs.into_iter()
        .filter_map(|r| match pdf.get(r) {
            Some(Object::Stream(s)) => Some((r.num, pdf.stream_data(&s).unwrap_or_default())),
            _ => None,
        })
        .collect()
}

fn same_op(a: &content::Op, b: &content::Op) -> bool {
    a.operator == b.operator
        && a.operands.len() == b.operands.len()
        && a.operands.iter().zip(&b.operands).all(|(x, y)| same(x, y))
        && match (&a.inline_image, &b.inline_image) {
            (Some(x), Some(y)) => {
                x.data == y.data
                    && same(&Object::Dict(x.dict.clone()), &Object::Dict(y.dict.clone()))
            }
            (None, None) => true,
            _ => false,
        }
}

/// Equal, numbers within what compact writing keeps.
fn same(a: &Object, b: &Object) -> bool {
    match (a, b) {
        (Object::Array(x), Object::Array(y)) => {
            x.len() == y.len() && x.iter().zip(y).all(|(p, q)| same(p, q))
        }
        (Object::Dict(x), Object::Dict(y)) => {
            x.len() == y.len()
                && x.iter()
                    .zip(y.iter())
                    .all(|((k1, v1), (k2, v2))| k1 == k2 && same(v1, v2))
        }
        _ => match (a.as_f64(), b.as_f64()) {
            (Some(x), Some(y)) => (x - y).abs() <= 1e-5_f64.max(x.abs() * 1e-4),
            _ => a == b,
        },
    }
}

/// FNV-1a, 64 bits.
fn fnv(data: &[u8]) -> u64 {
    data.iter().fold(0xcbf2_9ce4_8422_2325, |h, &b| {
        (h ^ u64::from(b)).wrapping_mul(0x0100_0000_01b3)
    })
}

/// Writes the file's page summary and its two rewrites; returns how the
/// rewrites read back here (`ok`, or what differs).
fn rewrite(
    pdf: &Pdf,
    bytes: &[u8],
    pages: &[Vec<(u32, Vec<u8>)>],
    out: &Path,
    name: &str,
) -> String {
    let stem: String = name
        .chars()
        .map(|c| {
            if c.is_ascii_alphanumeric() || c == '.' || c == '-' {
                c
            } else {
                '_'
            }
        })
        .collect();
    let summary = serde_json::json!({
        "file": name,
        "repaired": pdf.repaired(),
        "pages": pages
            .iter()
            .map(|streams| streams.iter().map(|(num, d)| serde_json::json!([num, d.len(), fnv(d).to_string()])).collect::<Vec<_>>())
            .collect::<Vec<_>>(),
    });
    std::fs::write(out.join(format!("{stem}.json")), summary.to_string()).unwrap();

    // Whole: every object but cross-reference and object streams.
    let objects: Vec<(ObjRef, Object)> = pdf
        .object_refs()
        .into_iter()
        .filter_map(|r| Some((r, pdf.get(r)?)))
        .filter(|(_, o)| !matches!(o, Object::Stream(s) if s.dict.is("Type", "XRef") || s.dict.is("Type", "ObjStm")))
        .collect();
    let full = write::file(&pdf.version(), &objects, pdf.trailer());
    std::fs::write(out.join(format!("{stem}.full.pdf")), &full).unwrap();

    // Incremental: the catalog changed and an object added.
    let root = pdf.trailer().get("Root").and_then(Object::as_ref);
    let mut update = Vec::new();
    if let (Some(root), Some(mut catalog)) = (root, pdf.catalog()) {
        catalog.set("AiEngineCheck", true);
        update.push((root, Object::Dict(catalog)));
    }
    let next = pdf.object_refs().last().map_or(1, |r| r.num + 1).max(
        pdf.trailer()
            .i64("Size")
            .and_then(|s| u32::try_from(s).ok())
            .unwrap_or(0),
    );
    update.push((
        ObjRef::new(next, 0),
        Object::String(b"added by the ai_engine corpus check".to_vec()),
    ));
    let incremental = write::incremental(bytes, &update, pdf.trailer());
    std::fs::write(out.join(format!("{stem}.incr.pdf")), &incremental).unwrap();

    let mut problems = Vec::new();
    for (label, rewritten) in [("full", full), ("incr", incremental)] {
        match Pdf::open(Arc::from(rewritten)) {
            Ok(again) => {
                let c = check(&again);
                if c.pages != pages {
                    problems.push(format!("{label}: pages differ"));
                }
                if label == "incr" && again.get(ObjRef::new(next, 0)).is_none() {
                    problems.push("incr: added object missing".into());
                }
            }
            Err(e) => problems.push(format!("{label}: {e}")),
        }
    }
    if problems.is_empty() {
        "ok".into()
    } else {
        problems.join("; ")
    }
}