Skip to content

Open-source manufacturing ERP, MES and QMS. Quoting, MRP, inventory, shop floor, quality and lot/serial traceability on one Postgres schema, with a REST API and MCP server. Self-host or use Carbon Cloud.

45.3 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-only
// Carbon (github.com/crbnos/carbon). Modified or adapted versions of this file,
// including ports, remain AGPLv3; serving them over a network requires releasing their source.

//! `optimize` action — any supported CAD/mesh input → optimised GLB. Resolves the
//! source format (explicit or content auto-detected), loads it into a GLB
//! (OCCT-tessellating B-rep, ingesting STL, mmapping a GLB, or repacking a text
//! glTF to GLB via a streaming base64 decode), then runs the
//! meshopt geometry passes + codec encode (via `crates/optimize`), walking a
//! simplify ladder until the render-weight + output-size gates pass. Async job;
//! uploads the `glb` output (plus the ingest outputs it was asked for: `raw`,
//! `thumbnail`, `pmi`) to their submit-time URLs. Fails loud with a typed error
//! (`unsupported_format` / `ambiguous_format` / `tessellation_failed` /
//! `cannot_fit_budget`) rather than storing an over-cap or wrong result.

use crate::formats::{self, Format};
use crate::jobs::{Done, Output};
use crate::{admission, compute, http, telemetry, AppState};
use serde_json::json;
use std::time::{Duration, Instant};

pub struct OptimizeReq {
    pub source_url: String,
    /// Declared source format, or `"auto"` / empty to content-detect.
    pub format: String,
    /// Storage path recorded in the completion pointer (the `glb` upload).
    pub glb_path: Option<String>,
    pub opts: Opts,
    /// Ingest: also write the retained raw's compact form (output `raw`), from
    /// the same download. `Some(path)` records where it goes.
    pub compact: Option<Option<String>>,
    /// Ingest: also render the preview thumbnail (output `thumbnail`) from the
    /// optimised GLB, still in memory.
    pub thumbnail: Option<ThumbnailOut>,
    /// Ingest: also read the model's PMI (output `pmi`, `pmi.json`) when the
    /// source is a STEP or BinXCAF. `Some(path)` records where it goes. A
    /// model with no PMI writes no output; a failed read is a warning.
    pub pmi: Option<Option<String>>,
}

/// The `pmi.json` an ingest read, with its counts for the result.
struct PmiOut {
    bytes: Vec<u8>,
    items: usize,
    views: usize,
}

/// Read the PMI of `load_path` (the BinXCAF the ingest made, or the source
/// itself). `source_format` is what was uploaded. `Ok(None)`: the model has
/// none, or is not a format that carries it (only STEP does; a BinXCAF is a
/// STEP the ingest already read).
fn read_ingest_pmi(
    load_path: &str,
    source_format: &str,
    source_path: &str,
) -> Result<Option<PmiOut>, String> {
    if source_format != "step" && source_format != "xbf" {
        return Ok(None);
    }
    // The STEP header names the unit the author drew in; a BinXCAF has none.
    let source_unit = if source_format == "step" {
        read_head_bytes(source_path, converter::graph::SOURCE_UNIT_SCAN_BYTES)
            .ok()
            .and_then(|head| {
                converter::pmi::pmi_source_unit(&converter::graph::detect_source_unit_bytes(&head))
            })
    } else {
        None
    };
    let Some(doc) = converter::pmi::extract_pmi(load_path, source_unit)? else {
        return Ok(None);
    };
    let bytes = converter::pmi::to_json(&doc)?;
    Ok(Some(PmiOut {
        bytes,
        items: doc.items.len(),
        views: doc.views.len(),
    }))
}

/// The thumbnail an ingest renders (see `thumbnail` for the camera).
pub struct ThumbnailOut {
    pub path: Option<String>,
    pub size: u32,
    pub direction: [f32; 3],
}

/// The retained raw's compact form.
struct Compacted {
    bytes: bytes::Bytes,
    /// `xbf` (a STEP as BinXCAF, then zstd) or `zstd` (the raw as it is).
    kind: &'static str,
}

/// What an ingest hands the optimiser: the file to tessellate, its format, and
/// the compacted raw. A STEP becomes BinXCAF once, and the optimiser reads that
/// (identical geometry, and far faster to parse) instead of parsing the STEP a
/// second time.
struct Ingest {
    load_path: String,
    declared: String,
    source_format: &'static str,
    /// The file to compress for the retained raw, and what it is. Compressed
    /// after the optimise, so its bytes are not held through it.
    compact_from: Option<(String, &'static str)>,
    xbf: Option<std::path::PathBuf>,
    warnings: Vec<String>,
}

fn prepare_ingest(path: &str, declared: &str, ext: Option<&str>) -> Result<Ingest, ActionErr> {
    let head = read_head_bytes(path, 512 * 1024).map_err(|e| ActionErr::new("invalid_input", e))?;
    let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
    let (format, _) = formats::resolve(declared, &head, size, ext)
        .map_err(|e| ActionErr::new(e.code, e.message))?;
    let mut ingest = Ingest {
        load_path: path.to_string(),
        declared: declared.to_string(),
        source_format: format.name(),
        compact_from: None,
        xbf: None,
        warnings: Vec::new(),
    };
    match format {
        // Already the compact form.
        Format::Xbf => {}
        Format::Step => {
            let xbf = http::temp_path("xbf");
            let xbf_str = xbf.to_string_lossy().to_string();
            match converter::convert::step_to_xbf(path, &xbf_str) {
                Ok(()) => {
                    ingest.compact_from = Some((xbf_str.clone(), "xbf"));
                    ingest.load_path = xbf_str;
                    ingest.declared = "xbf".into();
                    ingest.xbf = Some(xbf);
                }
                // The optimiser still reads the STEP; only the compact form is lost.
                Err(e) => {
                    let _ = std::fs::remove_file(&xbf);
                    ingest.warnings.push(format!("compact: {}", e.message));
                }
            }
        }
        _ => ingest.compact_from = Some((path.to_string(), "zstd")),
    }
    Ok(ingest)
}

/// Optimise, with the ingest outputs when asked for: the optimised GLB, the
/// compacted raw, and the thumbnail (or why it failed: a preview never fails
/// the job).
#[allow(clippy::type_complexity)]
fn run_ingest(
    path: &str,
    declared: &str,
    ext: Option<&str>,
    opts: &Opts,
    compact: bool,
    thumbnail: Option<(u32, [f32; 3])>,
    pmi: bool,
) -> Result<
    (
        Outcome,
        Option<Compacted>,
        Option<Result<Vec<u8>, String>>,
        Option<PmiOut>,
    ),
    ActionErr,
> {
    let ingest = if compact { Some(prepare_ingest(path, declared, ext)?) } else { None };
    let (load_path, load_declared) = match &ingest {
        Some(i) => (i.load_path.as_str(), i.declared.as_str()),
        None => (path, declared),
    };
    let outcome = run_optimize(load_path, load_declared, ext, opts);
    // The PMI is read from the same BinXCAF, before it is deleted, and only
    // once the optimise has succeeded. A failure is a warning: the preview
    // matters more than its annotations.
    let pmi_read = match (&outcome, pmi) {
        (Ok(o), true) => {
            let source_format = ingest.as_ref().map_or(o.detected_format, |i| i.source_format);
            Some(read_ingest_pmi(load_path, source_format, path))
        }
        _ => None,
    };
    // Compress the retained raw only once the optimise has succeeded, and
    // before the BinXCAF it may be made from is deleted.
    let compacted = match (&outcome, ingest.as_ref().and_then(|i| i.compact_from.as_ref())) {
        (Ok(_), Some((from, kind))) => Some(
            crate::actions::compact::zstd_file_mapped(from, crate::actions::compact::ZSTD_LEVEL)
                .map(|bytes| Compacted { bytes, kind }),
        ),
        _ => None,
    };
    if let Some(xbf) = ingest.as_ref().and_then(|i| i.xbf.as_ref()) {
        let _ = std::fs::remove_file(xbf);
    }
    let mut outcome = outcome?;
    if let Some(i) = ingest {
        outcome.detected_format = i.source_format;
        outcome.warnings.extend(i.warnings);
    }
    let compacted = match compacted {
        Some(Ok(c)) => Some(c),
        Some(Err(e)) => {
            outcome.warnings.push(format!("compact: {e}"));
            None
        }
        None => None,
    };
    let pmi_out = match pmi_read {
        Some(Ok(p)) => p,
        Some(Err(e)) => {
            outcome.warnings.push(format!("pmi: {e}"));
            None
        }
        None => None,
    };
    let png = thumbnail.map(|(size, direction)| {
        thumbnail::render_png(&outcome.glb, size, direction).map_err(|e| e.0)
    });
    Ok((outcome, compacted, png, pmi_out))
}

pub struct Opts {
    pub codec: optimize::Codec,
    /// Simplify rungs walked in order; the first passing the gates wins. `None` =
    /// full fidelity. Default `[None]`.
    pub ladder: Vec<Option<f32>>,
    pub simplify_aggressive: bool,
    pub weld: bool,
    pub reorder: bool,
    /// Quality/perf knob: max simplify error in mm (`None` = ratio-only).
    pub tolerance: Option<f32>,
    /// Auto-mode normalized error budget. `None` = lossless.
    pub auto_error: Option<f32>,
    /// The caller supplied no explicit quality knob, so the service owns the
    /// fidelity policy: `run_optimize` runs a lossless pass first and, only when
    /// a gate fails, re-runs with a keep ratio PREDICTED from that pass's
    /// measured overshoot (`required_keep`), ignoring `ladder`/`auto_error`.
    /// Off when any explicit knob was given — then the caller's ladder is
    /// authoritative.
    pub auto_scale: bool,
    /// Draco quantization bits (position, normal, texcoord) — Draco codec only.
    pub draco_bits: (i32, i32, i32),
    /// Quantize normals to i16 (none/meshopt codecs).
    pub quantize_normals: bool,
    /// Merge same-material primitives within a mesh.
    pub merge_primitives: bool,
    /// Decoded (render-weight) byte ceiling — the "packed" gate.
    pub max_packed: usize,
    /// Encoded output byte ceiling.
    pub max_output: usize,
    /// STEP/IGES tessellation deflection (ignored for mesh input).
    pub lin: f64,
    pub ang: f64,
    /// Wall-clock budget for the simplify ladder (`quality.time_budget_secs`,
    /// else auto: 720s on Lambda). `None` = unbounded. `spawn` turns this
    /// into an absolute `deadline` relative to job start (so it also charges the
    /// download + tessellation time already spent).
    pub budget: Option<Duration>,
    /// Absolute deadline computed in `spawn`; consumed by the ladder. Never set
    /// from JSON.
    pub deadline: Option<Instant>,
}

/// A typed action failure carrying the snake_case API error code.
#[derive(Debug)]
struct ActionErr {
    code: &'static str,
    message: String,
}
impl ActionErr {
    fn new(code: &'static str, message: impl Into<String>) -> Self {
        ActionErr {
            code,
            message: message.into(),
        }
    }
}

pub fn spawn(state: &AppState, job_id: &str, req: OptimizeReq) {
    let jobs = state.jobs.clone();
    let admission = state.admission.clone();
    let sources = state.sources.clone();
    let job_id = job_id.to_string();
    telemetry::spawn_job(&job_id.clone(), "optimize", async move {
        if jobs.is_canceled(&job_id).await {
            return;
        }
        let started = Instant::now();

        let src = match sources.download(&req.source_url, "model", None).await {
            Ok(src) => src,
            Err(e) => {
                eprintln!("[{job_id}] optimize failed: source download: {}", e.message);
                jobs.set_error(&job_id, &e.code, e.message).await;
                return;
            }
        };
        let tmp_str = src.path_string();
        let declared = req.format.clone();
        let ext = url_ext(&req.source_url);
        let mut opts = req.opts;
        let compact = req.compact.is_some();
        let thumbnail = req.thumbnail.as_ref().map(|t| (t.size, t.direction));
        let pmi = req.pmi.is_some();

        let res = {
            let grant = admission
                .acquire(admission::estimate_mb(http::file_len(src.path()).await))
                .await;
            // The wait may have been long: a job cancelled meanwhile is dropped
            // before it takes the memory it was granted.
            if jobs.is_canceled(&job_id).await {
                return;
            }
            // Running from here on: the wait for memory is queue time, not
            // work, so the budget starts now too.
            jobs.set_status(&job_id, "running").await;
            eprintln!(
                "[{job_id}] optimize running (codec={:?} format={declared})",
                opts.codec
            );
            opts.deadline = opts.budget.map(|b| Instant::now() + b);
            compute::run(Some(grant), move || {
                run_ingest(&tmp_str, &declared, ext.as_deref(), &opts, compact, thumbnail, pmi)
            })
            .await
        };
        drop(src);

        let (mut outcome, compacted, png, pmi_out) = match res {
            Ok(Ok(o)) => o,
            Ok(Err(e)) => {
                eprintln!("[{job_id}] optimize failed ({}): {}", e.code, e.message);
                jobs.set_error(&job_id, e.code, e.message).await;
                return;
            }
            Err(e) => {
                let msg = format!("optimize {e}");
                eprintln!("[{job_id}] {msg}");
                jobs.set_error(&job_id, e.code("optimize_failed"), msg).await;
                return;
            }
        };

        let optimise_ms = started.elapsed().as_millis() as i64;
        eprintln!(
            "[{job_id}] optimize done: {} ({} via) {} -> {} tris, {} -> {} bytes, ratio={:?}, {optimise_ms}ms",
            outcome.detected_format,
            outcome.detected_via,
            outcome.stats.input_triangles,
            outcome.stats.output_triangles,
            outcome.stats.input_bytes,
            outcome.glb.len(),
            outcome.ratio,
        );

        let png = match png {
            Some(Ok(png)) => Some(png),
            Some(Err(e)) => {
                outcome.warnings.push(format!("thumbnail: {e}"));
                None
            }
            None => None,
        };
        let mut result_outputs = json!({
            "glb": {
                "path": req.glb_path,
                "bytes": outcome.glb.len(),
                "codec": codec_name(outcome.codec),
            }
        });
        if let Some(c) = &compacted {
            result_outputs["raw"] = json!({
                "path": req.compact.clone().flatten(),
                "bytes": c.bytes.len(),
                "kind": c.kind,
            });
        }
        if let Some(png) = &png {
            result_outputs["thumbnail"] = json!({
                "path": req.thumbnail.as_ref().and_then(|t| t.path.clone()),
                "bytes": png.len(),
            });
        }
        if let Some(p) = &pmi_out {
            result_outputs["pmi"] = json!({
                "path": req.pmi.clone().flatten(),
                "bytes": p.bytes.len(),
                "items": p.items,
                "views": p.views,
            });
        }
        let done = Done {
            result: json!({
                "detected_format": outcome.detected_format,
                "detected_via": outcome.detected_via,
                "outputs": result_outputs,
                "warnings": outcome.warnings,
            }),
            stats: json!({
                "input_triangles": outcome.stats.input_triangles,
                "output_triangles": outcome.stats.output_triangles,
                "input_bytes": outcome.stats.input_bytes,
                "output_bytes": outcome.glb.len(),
                "render_weight_bytes": outcome.stats.decoded_bytes,
                "simplify_ratio_used": outcome.ratio,
                "optimise_ms": optimise_ms,
                "compact_bytes": compacted.as_ref().map(|c| c.bytes.len()),
                "thumbnail_bytes": png.as_ref().map(|p| p.len()),
                "pmi_bytes": pmi_out.as_ref().map(|p| p.bytes.len()),
            }),
        };
        let mut outputs = vec![Output {
            name: "glb".into(),
            content_type: "model/gltf-binary".into(),
            bytes: outcome.glb.into(),
        }];
        if let Some(c) = compacted {
            outputs.push(Output {
                name: "raw".into(),
                content_type: "application/zstd".into(),
                bytes: c.bytes,
            });
        }
        if let Some(png) = png {
            outputs.push(Output {
                name: "thumbnail".into(),
                content_type: "image/png".into(),
                bytes: png.into(),
            });
        }
        if let Some(p) = pmi_out {
            outputs.push(Output {
                name: "pmi".into(),
                content_type: "application/json".into(),
                bytes: p.bytes.into(),
            });
        }
        jobs.finish(&job_id, outputs, done, None).await;
    });
}

struct Outcome {
    glb: Vec<u8>,
    stats: optimize::Stats,
    codec: optimize::Codec,
    ratio: Option<f32>,
    warnings: Vec<String>,
    detected_format: &'static str,
    detected_via: &'static str,
}

/// GLB source bytes — an owned tessellation/STL-ingest, a memory-mapped uploaded
/// GLB, or a memory-mapped repacked temp GLB (from a text glTF). All OS-paged, off
/// the RSS.
enum Src {
    Owned(Vec<u8>),
    Mapped(memmap2::Mmap),
    /// Repacked temp `.glb` (from text glTF); the `TempPath` holds the file open
    /// for the mmap's lifetime and deletes it on drop.
    MappedTemp(memmap2::Mmap, #[allow(dead_code)] tempfile::TempPath),
}
impl Src {
    fn bytes(&self) -> &[u8] {
        match self {
            Src::Owned(v) => v,
            Src::Mapped(m) | Src::MappedTemp(m, _) => m,
        }
    }
}

/// Safety margin applied to the predicted keep ratio. Decoded (render-weight)
/// bytes are exactly linear in kept triangles and encoded bytes are close to
/// linear, but the meshopt encoder's per-vertex rate wobbles with connectivity —
/// the margin makes the first targeted pass land under the gates instead of a
/// hair over (which would cost another full pass).
const AUTO_TARGET_MARGIN: f64 = 0.85;

/// How many measured, targeted simplify passes to attempt after the lossless
/// pass fails the gates. Each pass re-predicts from the previous pass's real
/// numbers, so convergence is geometric; 3 misses means the prediction model
/// does not hold for this input (fail loudly rather than loop).
const AUTO_MAX_TARGETED_PASSES: usize = 3;

/// Length of a GLB's JSON (structure) chunk — the scene graph, nodes, and
/// materials. It does not shrink when triangles are removed, so the encoded
/// size is `json + bin(tris)`: an affine model, not a linear one. `None` on
/// anything that isn't a well-formed GLB (we only ever call this on our own
/// encoder's output, so that is a defensive fallback, not an expected path).
fn glb_json_len(glb: &[u8]) -> Option<usize> {
    if glb.len() < 20 || &glb[0..4] != b"glTF" || &glb[16..20] != b"JSON" {
        return None;
    }
    let len = u32::from_le_bytes(glb[12..16].try_into().ok()?) as usize;
    (len <= glb.len()).then_some(len)
}

/// Predict the fraction of triangles a failing pass may keep so the next pass
/// lands under both gates. Decoded (render-weight) bytes are linear in triangle
/// count; encoded bytes are the JSON structure constant plus a ~linear mesh
/// term — so the constant is subtracted from both the measurement and the cap
/// before inverting the worst overshoot (times the margin). `None` = the pass
/// already fits.
fn required_keep(
    decoded: usize,
    encoded: usize,
    json_len: usize,
    max_packed: usize,
    max_output: usize,
) -> Option<f64> {
    if decoded <= max_packed && encoded <= max_output {
        return None;
    }
    let over_packed = decoded as f64 / max_packed as f64;
    // Guard division by ~zero: json_len >= max_output is the infeasible case
    // the caller fails fast on; a stale/absent json_len degrades to the plain
    // linear model.
    let bin = encoded.saturating_sub(json_len) as f64;
    let bin_cap = max_output.saturating_sub(json_len) as f64;
    let over_encoded = if bin_cap > 0.0 {
        bin / bin_cap
    } else {
        f64::INFINITY
    };
    Some((AUTO_TARGET_MARGIN / over_packed.max(over_encoded)).min(1.0))
}

/// Resolve the format and load the source into GLB bytes, then fit it under the
/// size/render-weight gates.
///
/// Auto mode (no explicit quality knob) is MEASURE → PREDICT, never a guessed
/// coefficient: pass 1 is lossless (weld + reorder + encode only), and since
/// both gate metrics scale ~linearly with triangle count, its real numbers
/// predict the exact keep ratio a failing model needs (`required_keep`). Most
/// models fit lossless and are returned untouched — geometry is only reduced
/// when a gate proves it must be, and then by the minimum measured amount.
/// (The old weight-derived error budget decimated models that had no size
/// problem: a 34 MB-decoded connector — 12x under the render-weight gate —
/// lost 95% of its triangles, pins first, because meshopt's error normalizes
/// to the merged mesh's extent.)
///
/// Explicit knobs (ladder/simplify/tolerance/auto_error) bypass all of this —
/// the caller owns the policy and their ladder is walked as given. If nothing
/// fits, fail with `cannot_fit_budget` (never store an over-cap blob).
fn run_optimize(
    path: &str,
    declared: &str,
    ext: Option<&str>,
    opts: &Opts,
) -> Result<Outcome, ActionErr> {
    let head = read_head_bytes(path, 512 * 1024).map_err(|e| ActionErr::new("invalid_input", e))?;
    let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);

    let (format, detected_via) = formats::resolve(declared, &head, size, ext)
        .map_err(|e| ActionErr::new(e.code, e.message))?;
    let detected_format = format.name();

    let src = load_source(path, format, &head, opts)?;
    let glb = src.bytes();
    let input_bytes = glb.len();

    // One optimize pass over the tessellated GLB. Every source is GLB here
    // (STEP/STL ingested, GLB mmap'd, text glTF repacked in load_source).
    let run_pass = |simplify: Option<f32>,
                    auto_error: Option<f32>,
                    aggressive: bool|
     -> Result<Outcome, ActionErr> {
        let o = optimize::Options {
            codec: opts.codec,
            simplify,
            tolerance: opts.tolerance,
            auto_error,
            simplify_aggressive: aggressive,
            draco_bits: opts.draco_bits,
            quantize_normals: opts.quantize_normals,
            merge_primitives: opts.merge_primitives,
            weld: opts.weld,
            reorder: opts.reorder,
        };
        let mut res = optimize::optimize_glb(glb, &o)
            .map_err(|e| ActionErr::new("optimize_failed", e.message))?;
        res.stats.input_bytes = input_bytes;
        Ok(Outcome {
            glb: res.glb,
            stats: res.stats,
            codec: opts.codec,
            ratio: simplify,
            warnings: Vec::new(),
            detected_format,
            detected_via,
        })
    };
    let fits =
        |o: &Outcome| o.stats.decoded_bytes <= opts.max_packed && o.glb.len() <= opts.max_output;
    let over_msg = |label: &str, o: &Outcome| {
        format!(
            "{label} over budget: render-weight {}B (max {}B), output {}B (max {}B)",
            o.stats.decoded_bytes,
            opts.max_packed,
            o.glb.len(),
            opts.max_output
        )
    };

    if opts.auto_scale {
        // Measure at full fidelity first.
        let lossless = run_pass(None, None, false)?;
        if fits(&lossless) {
            return Ok(lossless);
        }
        let mut warnings = vec![over_msg("lossless", &lossless)];
        // The JSON (structure) chunk survives simplification whole — if it
        // alone exceeds the output cap, no amount of triangle removal can ever
        // fit. Fail now with the real reason instead of burning three passes
        // discovering it. (Revisit if the optimizer ever learns to prune the
        // scene graph itself — then this floor stops being a floor.)
        let json_len = glb_json_len(&lossless.glb).unwrap_or(0);
        if json_len >= opts.max_output {
            return Err(ActionErr::new(
                "cannot_fit_budget",
                format!(
                    "scene structure alone ({json_len}B of glTF JSON) exceeds the output cap \
                     ({}B) — simplification cannot fit this model",
                    opts.max_output
                ),
            ));
        }
        // Predict the keep ratio from the measured overshoot, then converge:
        // each miss re-predicts from that pass's real numbers, so the ratio
        // shrinks geometrically. The bounded pass count replaces the old
        // skip-to-coarsest deadline dance — pass 2 already targets the size the
        // model must reach, there are no intermediate rungs to skip.
        let mut keep = required_keep(
            lossless.stats.decoded_bytes,
            lossless.glb.len(),
            json_len,
            opts.max_packed,
            opts.max_output,
        )
        .unwrap_or(1.0);
        let mut smallest = lossless.glb.len();
        let mut prev_encoded = lossless.glb.len();
        // Only its sizes are needed from here: free the full-fidelity model
        // before the targeted passes build theirs.
        drop(lossless);
        let mut aggressive = false;
        for _ in 0..AUTO_MAX_TARGETED_PASSES {
            // Ratio targeting hits its triangle count whatever the error — the
            // gate has proven the reduction unavoidable, and meshopt removes the
            // lowest-error collapses first, so what fidelity can survive does.
            let out = run_pass(Some(keep as f32), None, aggressive)?;
            if fits(&out) {
                return Ok(out);
            }
            warnings.push(over_msg(
                &format!("keep={keep:.4}{}", if aggressive { " (sloppy)" } else { "" }),
                &out,
            ));
            let further = required_keep(
                out.stats.decoded_bytes,
                out.glb.len(),
                glb_json_len(&out.glb).unwrap_or(0),
                opts.max_packed,
                opts.max_output,
            )
            .unwrap_or(1.0);
            // Re-anchor on what the pass actually ACHIEVED, not what it was
            // asked for — topology-preserving simplify stalls above the target
            // when it runs out of legal edge collapses, and multiplying the
            // requested ratio down further would change nothing.
            let achieved = out.stats.output_triangles as f64
                / out.stats.input_triangles.max(1) as f64;
            keep = (achieved * further).min(keep);
            // Stall: the pass barely moved the encoded size — no smaller target
            // can help while collapses are exhausted. Escalate to sloppy
            // (positional) simplification, which trades topology for the
            // ability to reach any triangle count. Only ever reached after
            // bounded simplification has provably hit its floor.
            if out.glb.len() as f64 > prev_encoded as f64 * 0.98 {
                aggressive = true;
            }
            prev_encoded = out.glb.len();
            smallest = smallest.min(out.glb.len());
        }
        return Err(ActionErr::new(
            "cannot_fit_budget",
            format!(
                "could not fit the size/render-weight budget (smallest {smallest}B, max {}B) \
                 after {AUTO_MAX_TARGETED_PASSES} targeted passes. {}",
                opts.max_output,
                warnings.join("; ")
            ),
        ));
    }

    // Explicit quality knobs: walk the caller's ladder as given, first rung
    // under both gates wins.
    let ladder = if opts.ladder.is_empty() {
        vec![None]
    } else {
        opts.ladder.clone()
    };
    // Only the last rung's size is reported; its model is not kept.
    let mut last_len: Option<usize> = None;
    let mut warnings: Vec<String> = Vec::new();
    let mut i = 0;
    while i < ladder.len() {
        // Time-budget gate: if the wall-clock budget is spent and coarser rungs
        // remain, jump straight to the coarsest. The skipped middle rungs are
        // finer (larger output) than the coarsest, so if they'd fail the size gate
        // the coarsest is the only one with a chance — running them first only
        // burns the remaining window. Preserves the size invariant: a no-fit
        // coarsest still fails `cannot_fit_budget` below (never stores over-cap).
        if let Some(dl) = opts.deadline {
            let last = ladder.len() - 1;
            if i < last && Instant::now() >= dl {
                warnings.push(format!(
                    "time budget spent after rung {:?}; skipping to coarsest rung {:?}",
                    ladder[i], ladder[last]
                ));
                i = last;
            }
        }
        let rung = ladder[i];
        let outcome = run_pass(rung, opts.auto_error, opts.simplify_aggressive)?;
        if fits(&outcome) {
            return Ok(outcome);
        }
        warnings.push(over_msg(&format!("rung {rung:?}"), &outcome));
        last_len = Some(outcome.glb.len());
        i += 1;
    }

    // Nothing fit the budget even at the most aggressive rung: fail loud rather
    // than store an artifact the served bucket would reject.
    let smallest = last_len.unwrap_or(0);
    Err(ActionErr::new(
        "cannot_fit_budget",
        format!(
            "could not fit the size/render-weight budget (smallest {smallest}B, max {}B); tried rungs {:?}. {}",
            opts.max_output,
            ladder,
            warnings.join("; ")
        ),
    ))
}

/// The downloaded source, memory-mapped: the page cache holds it, not the
/// heap, while it is parsed.
fn map_source(path: &str) -> Result<memmap2::Mmap, ActionErr> {
    let file = std::fs::File::open(path)
        .map_err(|e| ActionErr::new("invalid_input", format!("open source: {e}")))?;
    // SAFETY: the source is this job's own temp file; nothing writes it while
    // it is mapped.
    unsafe { memmap2::Mmap::map(&file) }
        .map_err(|e| ActionErr::new("invalid_input", format!("mmap source: {e}")))
}

/// Load the resolved format into GLB bytes.
fn load_source(path: &str, format: Format, head: &[u8], opts: &Opts) -> Result<Src, ActionErr> {
    match format {
        Format::Step => {
            let glb = converter::convert::convert_step_head(path, head, opts.lin, opts.ang)
                .map_err(|e| ActionErr::new("tessellation_failed", e.message))?
                .glb;
            Ok(Src::Owned(glb))
        }
        Format::Xbf => {
            // Compacted retained raw (BinXCAF). Geometry already mm; tessellate
            // like STEP. Lets reoptimise run off the compacted source.
            let glb = converter::convert::convert_xbf(path, opts.lin, opts.ang)
                .map_err(|e| ActionErr::new("tessellation_failed", e.message))?
                .glb;
            Ok(Src::Owned(glb))
        }
        Format::Stl => {
            let bytes = map_source(path)?;
            let glb = optimize::stl_to_glb(&bytes)
                .map_err(|e| ActionErr::new("optimize_failed", e.message))?;
            Ok(Src::Owned(glb))
        }
        Format::Glb => Ok(Src::Mapped(map_source(path)?)),
        Format::Gltf => {
            use std::io::Write as _;
            // Repack text glTF → GLB with a streaming base64 decode, then mmap the
            // GLB and run the bounded GLB path. A serde parse of a multi-GB glTF
            // holds the base64 string + decoded bytes at once (~3× the file); this
            // keeps peak memory off the geometry (it lives in the temp GLB on disk).
            let gltf = map_source(path)?;
            // `geometry-*`, so startup cleanup removes it after a kill.
            let mut tmp = tempfile::Builder::new()
                .prefix("geometry-")
                .tempfile()
                .map_err(|e| ActionErr::new("invalid_input", format!("temp glb: {e}")))?;
            {
                let mut bw = std::io::BufWriter::new(&mut tmp);
                optimize::gltf_to_glb(&gltf, &mut bw)
                    .map_err(|e| ActionErr::new("optimize_failed", e.message))?;
                bw.flush().map_err(|e| {
                    ActionErr::new("optimize_failed", format!("flush temp glb: {e}"))
                })?;
            }
            let temp_path = tmp.into_temp_path();
            let glb_file = std::fs::File::open(&temp_path)
                .map_err(|e| ActionErr::new("invalid_input", format!("open temp glb: {e}")))?;
            let glb_map = unsafe { memmap2::Mmap::map(&glb_file) }
                .map_err(|e| ActionErr::new("invalid_input", format!("mmap temp glb: {e}")))?;
            Ok(Src::MappedTemp(glb_map, temp_path))
        }
        // Plain mesh formats: parse → triangle-soup GLB (the optimiser's weld
        // pass reconstructs sharing) → the bounded GLB path.
        Format::Obj | Format::Ply | Format::Off | Format::Bim | Format::ThreeMf | Format::Amf => {
            let bytes = map_source(path)?;
            let glb = match format {
                Format::Obj => optimize::obj_to_glb(&bytes),
                Format::Ply => optimize::ply_to_glb(&bytes),
                Format::Off => optimize::off_to_glb(&bytes),
                Format::Bim => optimize::bim_to_glb(&bytes),
                Format::ThreeMf => optimize::threemf_to_glb(&bytes),
                _ => optimize::amf_to_glb(&bytes),
            }
            .map_err(|e| ActionErr::new("invalid_input", e.message))?;
            Ok(Src::Owned(glb))
        }
        // Exact B-rep sources tessellated by OCCT, same walk as STEP.
        Format::Iges => {
            let glb = converter::convert::convert_iges(path, opts.lin, opts.ang)
                .map_err(|e| ActionErr::new("tessellation_failed", e.message))?
                .glb;
            Ok(Src::Owned(glb))
        }
        Format::Brep => {
            let glb = converter::convert::convert_brep(path, opts.lin, opts.ang)
                .map_err(|e| ActionErr::new("tessellation_failed", e.message))?
                .glb;
            Ok(Src::Owned(glb))
        }
    }
}

fn read_head_bytes(path: &str, cap: usize) -> Result<Vec<u8>, String> {
    use std::io::Read;
    let file = std::fs::File::open(path).map_err(|e| format!("read temp: {e}"))?;
    let mut buf = Vec::new();
    file.take(cap as u64)
        .read_to_end(&mut buf)
        .map_err(|e| format!("read temp: {e}"))?;
    Ok(buf)
}

/// Filename extension hint from a (possibly signed) URL — the last-resort tiebreak
/// for format detection. Strips the query string.
fn url_ext(url: &str) -> Option<String> {
    let path = url.split(['?', '#']).next().unwrap_or(url);
    let name = path.rsplit('/').next().unwrap_or(path);
    name.rsplit_once('.')
        .map(|(_, ext)| ext.to_ascii_lowercase())
}

fn codec_name(c: optimize::Codec) -> &'static str {
    match c {
        optimize::Codec::None => "none",
        optimize::Codec::Meshopt => "meshopt",
        optimize::Codec::Draco => "draco",
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn default_opts() -> Opts {
        crate::request::optimize_opts(&serde_json::json!({}), &serde_json::json!({}))
    }

    fn fixture_step(name: &str, boxes: u32) -> String {
        let dir = std::env::temp_dir().join(format!("carbon-ingest-{name}"));
        std::fs::create_dir_all(&dir).unwrap();
        let path = dir.join("fixture.step").to_string_lossy().to_string();
        assert!(occt_bridge::write_test_step(&path, boxes), "write test STEP");
        path
    }

    #[test]
    fn ingest_of_a_step_compacts_to_xbf_and_matches_a_plain_optimise() {
        let step = fixture_step("step", 3);
        let opts = default_opts();

        let plain = run_optimize(&step, "step", Some("step"), &opts).expect("plain optimise");
        let (outcome, compacted, png, pmi) = run_ingest(
            &step,
            "step",
            Some("step"),
            &opts,
            true,
            Some((64, thumbnail::DEFAULT_DIRECTION)),
            true,
        )
        .expect("ingest");

        // The source is reported as what was uploaded, not the xbf read.
        assert_eq!(outcome.detected_format, "step");
        // Tessellated from the xbf: the same mesh as from the STEP.
        assert_eq!(
            outcome.stats.output_triangles,
            plain.stats.output_triangles
        );
        let compacted = compacted.expect("compacted raw");
        assert_eq!(compacted.kind, "xbf");
        let xbf = zstd::decode_all(compacted.bytes.as_ref()).unwrap();
        assert!(xbf.starts_with(b"BINFILE"), "the compact form is BinXCAF");
        let png = png.expect("thumbnail asked for").expect("thumbnail rendered");
        assert!(png.starts_with(b"\x89PNG"));
        assert!(pmi.is_none(), "plain boxes carry no PMI");
        assert!(
            outcome.warnings.iter().all(|w| !w.starts_with("pmi:")),
            "{:?}",
            outcome.warnings
        );
    }

    #[test]
    fn ingest_of_an_ap242_step_writes_its_pmi() {
        let dir = std::env::temp_dir().join("carbon-ingest-pmi");
        std::fs::create_dir_all(&dir).unwrap();
        let step = dir.join("fixture.step").to_string_lossy().to_string();
        assert!(occt_bridge::write_test_pmi_step(&step), "write AP242 fixture");
        let (outcome, _, _, pmi) =
            run_ingest(&step, "step", Some("step"), &default_opts(), true, None, true)
                .expect("ingest");
        let pmi = pmi.expect("the fixture has PMI");
        assert_eq!(pmi.items, 3);
        let doc: serde_json::Value = serde_json::from_slice(&pmi.bytes).unwrap();
        assert_eq!(doc["version"], 1);
        assert_eq!(doc["sourceUnit"], "mm");
        assert_eq!(doc["items"][0]["dimension"]["value"], 100.0);
        assert!(outcome.warnings.is_empty(), "{:?}", outcome.warnings);

        // Not asked for: not read.
        let (_, _, _, none) =
            run_ingest(&step, "step", Some("step"), &default_opts(), true, None, false)
                .expect("ingest");
        assert!(none.is_none());
    }

    #[test]
    fn ingest_of_a_mesh_compacts_its_bytes_as_they_are() {
        let step = fixture_step("mesh", 2);
        let opts = default_opts();
        // A plain (unquantized) GLB, as an upload would be.
        let dir = std::env::temp_dir().join("carbon-ingest-mesh");
        let xbf = dir.join("fixture.xbf").to_string_lossy().to_string();
        converter::convert::step_to_xbf(&step, &xbf).unwrap();
        let glb = converter::convert::convert_xbf(&xbf, opts.lin, opts.ang).unwrap().glb;
        let glb_path = dir.join("model.glb");
        std::fs::write(&glb_path, &glb).unwrap();
        let glb_str = glb_path.to_string_lossy().to_string();

        let (outcome, compacted, png, pmi) =
            run_ingest(&glb_str, "auto", Some("glb"), &opts, true, None, true).expect("ingest");
        // A mesh has no PMI to read, and asking is not a warning.
        assert!(pmi.is_none());
        assert!(outcome.warnings.iter().all(|w| !w.starts_with("pmi:")));

        let compacted = compacted.expect("compacted raw");
        assert_eq!(compacted.kind, "zstd");
        assert_eq!(zstd::decode_all(compacted.bytes.as_ref()).unwrap(), glb);
        assert!(png.is_none(), "no thumbnail asked for");
    }

    #[test]
    fn a_plain_optimise_writes_no_ingest_outputs() {
        let step = fixture_step("plain", 1);
        let (_, compacted, png, pmi) =
            run_ingest(&step, "step", Some("step"), &default_opts(), false, None, false).unwrap();
        assert!(pmi.is_none());
        assert!(compacted.is_none());
        assert!(png.is_none());
    }

    #[test]
    #[ignore = "scratch probe: needs real STEP files in .ai/scratch/"]
    fn probe_real_model() {
        let scratch = concat!(env!("CARGO_MANIFEST_DIR"), "/../../.ai/scratch");
        let opts = crate::request::optimize_opts(&serde_json::json!({}), &serde_json::json!({}));
        eprintln!(
            "PROBE opts lin={} ang={} auto_scale={} max_packed={} max_output={}",
            opts.lin, opts.ang, opts.auto_scale, opts.max_packed, opts.max_output
        );
        let mut files: Vec<_> = std::fs::read_dir(scratch)
            .expect("scratch dir")
            .filter_map(|e| e.ok().map(|e| e.path()))
            .filter(|p| {
                p.extension()
                    .and_then(|x| x.to_str())
                    .is_some_and(|x| ["step", "stp", "gltf", "glb", "stl"].contains(&x))
            })
            .collect();
        files.sort();
        assert!(!files.is_empty(), "no model files in {scratch}");
        for path in files {
            let p = path.to_str().unwrap();
            let name = path.file_name().unwrap().to_str().unwrap();
            let ext = path.extension().unwrap().to_str().unwrap();
            for (label, max_packed, max_output) in [
                ("prod-gates", opts.max_packed, opts.max_output),
                // Force the measured-overshoot predictor path.
                ("tight-gates", 8 * 1024 * 1024, 2 * 1024 * 1024),
            ] {
                let mut o = crate::request::optimize_opts(&serde_json::json!({}), &serde_json::json!({}));
                o.max_packed = max_packed;
                o.max_output = max_output;
                let t0 = std::time::Instant::now();
                let out = match run_optimize(p, ext, Some(ext), &o) {
                    Ok(r) => r,
                    Err(e) => panic!("optimize failed on {p} ({label}): {} {}", e.code, e.message),
                };
                eprintln!(
                    "PROBE {name} {label} in {:?}: ratio={:?} in_tris={} out_tris={} kept={:.1}% decoded={} glb={}",
                    t0.elapsed(),
                    out.ratio,
                    out.stats.input_triangles,
                    out.stats.output_triangles,
                    100.0 * out.stats.output_triangles as f64
                        / out.stats.input_triangles.max(1) as f64,
                    out.stats.decoded_bytes,
                    out.glb.len()
                );
            }
        }
    }

    /// A model under both gates must never be touched — `required_keep` returns
    /// `None` (no simplify pass runs at all). A weight-derived budget is what
    /// shredded the connector housing: 95% of its triangles deleted while it sat
    /// 5x under the output cap and 12x under the render-weight cap.
    #[test]
    fn required_keep_is_none_when_the_model_fits() {
        assert_eq!(required_keep(0, 0, 0, 100, 100), None);
        assert_eq!(required_keep(100, 100, 0, 100, 100), None);
        // The real connector's numbers vs the production gates.
        assert_eq!(
            required_keep(33_933_600, 9_957_924, 0, 419_430_400, 52_428_800),
            None
        );
    }

    /// The keep ratio is the inverse of the WORST gate overshoot times the
    /// safety margin — both gate metrics are ~linear in triangle count, so this
    /// is a prediction, not a guess.
    #[test]
    fn required_keep_tracks_the_worst_overshoot() {
        // 2x over the render-weight gate, encoded fine → keep 0.85/2.
        let k = required_keep(200, 10, 0, 100, 100).unwrap();
        assert!((k - 0.425).abs() < 1e-9, "got {k}");
        // Encoded gate is the worse one (4x) → it wins over decoded (2x).
        let k = required_keep(200, 400, 0, 100, 100).unwrap();
        assert!((k - 0.2125).abs() < 1e-9, "got {k}");
        // Barely over still shrinks (margin applies) and never exceeds 1.
        let k = required_keep(101, 0, 0, 100, 100).unwrap();
        assert!(k < 1.0 && k > 0.8, "got {k}");
    }

    /// Encoded bytes are structure + mesh; only the mesh term shrinks with
    /// triangles. The affine correction subtracts the JSON constant from both
    /// sides — without it the prediction is too optimistic on structure-heavy
    /// files (Vehicle: ~3 MB of scene JSON cost an extra full pass).
    #[test]
    fn required_keep_subtracts_the_structure_constant() {
        // encoded = 40 JSON + 360 mesh, cap 100: plain model says keep
        // 0.85/4 = 0.2125, but the mesh must fit in 100-40=60 → 0.85*60/360.
        let k = required_keep(0, 400, 40, 1_000_000, 100).unwrap();
        assert!((k - 0.85 * 60.0 / 360.0).abs() < 1e-9, "got {k}");
        // json_len at/over the cap is the caller's fast-fail case; the
        // prediction itself degrades to "keep nothing" rather than dividing
        // by zero.
        let k = required_keep(0, 400, 100, 1_000_000, 100).unwrap();
        assert!(k < 1e-12, "got {k}");
    }

    /// Convergence: re-predicting from a missed pass's real numbers shrinks the
    /// ratio geometrically — a 100x-over model reaches its target in two steps.
    #[test]
    fn required_keep_converges_geometrically() {
        let k1 = required_keep(10_000, 0, 0, 100, 100).unwrap(); // 100x over
        let after_pass = (10_000.0 * k1) as usize; // linear model holds
        assert!(after_pass <= 100, "one predicted pass should land under");
    }

    /// Header parse of our own encoder's GLB output, with the defensive
    /// fallbacks for anything malformed.
    #[test]
    fn glb_json_len_parses_and_guards() {
        let mut glb = Vec::new();
        glb.extend_from_slice(b"glTF");
        glb.extend_from_slice(&2u32.to_le_bytes()); // version
        glb.extend_from_slice(&28u32.to_le_bytes()); // total length
        glb.extend_from_slice(&8u32.to_le_bytes()); // JSON chunk length
        glb.extend_from_slice(b"JSON");
        glb.extend_from_slice(b"{\"a\":1} ");
        assert_eq!(glb_json_len(&glb), Some(8));
        assert_eq!(glb_json_len(b"not a glb"), None);
        assert_eq!(glb_json_len(&glb[..12]), None);
        // Declared JSON length larger than the blob → reject.
        let mut bad = glb.clone();
        bad[12..16].copy_from_slice(&10_000u32.to_le_bytes());
        assert_eq!(glb_json_len(&bad), None);
    }
}