Open-source manufacturing ERP, MES and QMS. Quoting, MRP, inventory, shop floor, quality and lot/serial traceability on one Postgres schema, with a REST API and MCP server. Self-host or use Carbon Cloud.
TypeScript ai-agentsbill-of-materialserpinventory-managementleanmanufacturingmanufacturing-execution-systemmcp-servermesmrpopen-source-erppostgresqlproduction-planningqmsquality-management-systemreact-routerself-hostedsupabasetraceabilitytypescript
// SPDX-License-Identifier: AGPL-3.0-only
// Carbon (github.com/crbnos/carbon). Modified or adapted versions of this file,
// including ports, remain AGPLv3; serving them over a network requires releasing their source.
//! Signed-URL download/upload + temp files — port of `_download`/`_upload`.
use crate::config;
use crate::error::ApiError;
use crate::telemetry;
use std::path::PathBuf;
use std::sync::atomic::{AtomicU64, Ordering};
static COUNTER: AtomicU64 = AtomicU64::new(0);
pub fn temp_path(ext: &str) -> PathBuf {
let n = COUNTER.fetch_add(1, Ordering::Relaxed);
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
std::env::temp_dir().join(format!("geometry-{}-{n}-{nanos}.{ext}", std::process::id()))
}
/// Delete the sources an earlier process left behind. A killed process (an OOM
/// kill above all) skips its actions' own cleanup, and a pod's temp volume
/// outlives the container restart, so they would pile up. Only for the standing
/// server at startup, before it has a job of its own.
pub fn clear_stale_sources() {
clear_sources_in(&std::env::temp_dir());
}
fn clear_sources_in(dir: &std::path::Path) {
let Ok(entries) = std::fs::read_dir(dir) else {
return;
};
for entry in entries.flatten() {
if entry.file_name().to_string_lossy().starts_with("geometry-") {
let _ = std::fs::remove_file(entry.path());
}
}
}
/// The whole download, headers to last byte. The read timeout below catches a
/// stalled connection; this catches one that trickles. Generous: a 2.5 GB
/// source at 1 MB/s still lands.
const DOWNLOAD_DEADLINE: std::time::Duration = std::time::Duration::from_secs(60 * 60);
/// The whole upload: two minutes plus 2 MB/s of body.
fn upload_deadline(bytes: Option<u64>) -> std::time::Duration {
std::time::Duration::from_secs(120 + bytes.unwrap_or(0) / (2 * 1024 * 1024))
}
/// One shared client for the process: reuses connections (keep-alive/h2)
/// instead of paying a fresh pool + TLS handshake per download/upload.
///
/// Follows no redirects: every URL here is a signed storage URL or our own
/// callback, checked by `config::validate_url`, and a redirect would take the
/// request to a host that was never checked. Errors never carry the URL
/// (`without_url`): signed URLs hold their token in the query string, and an
/// error message is logged, stored on the job and returned to the caller.
fn client() -> &'static reqwest::Client {
static CLIENT: std::sync::OnceLock<reqwest::Client> = std::sync::OnceLock::new();
CLIENT.get_or_init(|| {
reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none())
.danger_accept_invalid_certs(!config::verify_tls())
.connect_timeout(std::time::Duration::from_secs(10))
// Idle-read timeout, not a total deadline: a large source can stream
// for a while, but a stalled connection (no bytes for 60s) must fail
// so it can't hold a concurrency slot forever.
.read_timeout(std::time::Duration::from_secs(60))
.build()
.expect("reqwest client")
})
}
/// Resolves a host to its public addresses only, refusing one that has none.
/// `validate_url` sees only the URL, so a hostname that resolves to a private
/// address (a tenant URL pointed inside the network) passed it.
struct PublicOnly;
impl reqwest::dns::Resolve for PublicOnly {
fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving {
let host = name.as_str().to_string();
Box::pin(async move {
let addrs = public_addrs(&host).await?;
Ok(Box::new(addrs.into_iter()) as reqwest::dns::Addrs)
})
}
}
async fn public_addrs(
host: &str,
) -> Result<Vec<std::net::SocketAddr>, Box<dyn std::error::Error + Send + Sync>> {
let addrs: Vec<_> = tokio::net::lookup_host((host, 0))
.await?
.filter(|addr| config::is_public_ip(addr.ip()))
.collect();
if addrs.is_empty() {
return Err(format!("{host} resolves to no public address").into());
}
Ok(addrs)
}
/// The `fetch` action's client: it follows redirects (a CAD system may answer
/// a download with one to its object store), but only to URLs that pass
/// `config::validate_url`, and at most five. reqwest drops the Authorization
/// header on a redirect to another host. Outside dev mode it connects only to
/// public addresses (`PublicOnly`): `fetch` is the one action whose URL comes
/// from a third party (a CAD system's tenant URL is the customer's to set),
/// while every other URL is one of our own signed storage URLs.
fn fetch_client() -> &'static reqwest::Client {
static CLIENT: std::sync::OnceLock<reqwest::Client> = std::sync::OnceLock::new();
CLIENT.get_or_init(|| {
let policy = reqwest::redirect::Policy::custom(|attempt| {
if attempt.previous().len() >= 5 {
attempt.error("too many redirects")
} else if config::validate_url(attempt.url().as_str()).is_err() {
attempt.error("redirect to a host that is not allowed")
} else {
attempt.follow()
}
});
let builder = reqwest::Client::builder();
let builder = if config::require_https() {
builder.dns_resolver(std::sync::Arc::new(PublicOnly))
} else {
builder
};
builder
.redirect(policy)
.danger_accept_invalid_certs(!config::verify_tls())
.connect_timeout(std::time::Duration::from_secs(10))
.read_timeout(std::time::Duration::from_secs(60))
.build()
.expect("reqwest client")
})
}
/// What a download may do beyond the default (the `fetch` action's).
#[derive(Default)]
pub struct DownloadOpts<'a> {
/// Sent as the `Authorization` header, marked sensitive.
pub authorization: Option<&'a str>,
/// Refuse a body past this many bytes (413 `LIMIT_EXCEEDED`).
pub max_bytes: Option<u64>,
/// Write the bytes exactly as served: no zstd decoding, and follow
/// validated redirects.
pub passthrough: bool,
}
/// Stream the source to disk, hashing as it flows — one pass, no full-body RAM
/// buffer, and the content hash comes out free for the result-cache key.
/// `progress` (when given) is ticked with bytes done/total for live status.
///
/// Transparent zstd: a source whose first bytes are the zstd magic
/// (`28 B5 2F FD`) is streamed through a decoder, so a stored `.zst` (e.g. the
/// compacted `raw.xbf.zst`) lands on disk as the real STEP/XBF/mesh and every
/// action consumes it unchanged. The hash + size guard are over the
/// DECOMPRESSED bytes, so the result-cache key tracks geometry, not container.
pub async fn download_with(
url: &str,
dest: &std::path::Path,
progress: Option<&crate::progress::JobProgress>,
opts: &DownloadOpts<'_>,
) -> Result<u128, ApiError> {
let mut request = if opts.passthrough {
fetch_client().get(url)
} else {
client().get(url)
};
if let Some(auth) = opts.authorization {
let mut value = reqwest::header::HeaderValue::from_str(auth)
.map_err(|_| ApiError::invalid("invalid source.authorization"))?;
value.set_sensitive(true);
request = request.header(reqwest::header::AUTHORIZATION, value);
}
let limit = opts
.max_bytes
.map_or(config::max_source_bytes(), |b| b as usize);
let span = if opts.passthrough { "fetch source" } else { "download source" };
let download = download(request.timeout(DOWNLOAD_DEADLINE), dest, progress, limit, !opts.passthrough);
telemetry::outbound(span, url, unless_canceled(download)).await
}
/// Run a transfer for the current job, giving up the moment the job is
/// canceled (`jobs::track`'s token): a canceled job stops moving bytes.
async fn unless_canceled<T>(
transfer: impl std::future::Future<Output = Result<T, ApiError>>,
) -> Result<T, ApiError> {
let Some(token) = crate::jobs::current_cancel() else {
return transfer.await;
};
tokio::select! {
result = transfer => result,
_ = token.cancelled() => Err(ApiError::new(409, "canceled", "the job was canceled")),
}
}
/// `limit` 0 = unlimited.
async fn download(
request: reqwest::RequestBuilder,
dest: &std::path::Path,
progress: Option<&crate::progress::JobProgress>,
limit: usize,
decode_zstd: bool,
) -> Result<u128, ApiError> {
use async_compression::tokio::bufread::ZstdDecoder;
use futures_util::StreamExt;
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt};
let too_large = || ApiError::new(413, "LIMIT_EXCEEDED", "source file exceeds the size limit");
let resp = request
.send()
.await
.map_err(|e| {
ApiError::new(
422,
"READ_FAILED",
format!("could not download source: {}", e.without_url()),
)
})?;
if !resp.status().is_success() {
return Err(ApiError::new(
422,
"READ_FAILED",
format!("could not download source: {}", resp.status()),
));
}
if limit > 0 && resp.content_length().is_some_and(|len| len > limit as u64) {
return Err(too_large());
}
// content-length is the on-the-wire (possibly compressed) size — advisory for
// progress only; the real cap is the decompressed `written` counter below.
if let Some(p) = progress {
if let Some(len) = resp.content_length() {
p.total.store(len, std::sync::atomic::Ordering::Relaxed);
}
}
let read_err = |e: std::io::Error, dest: &std::path::Path| {
let dest = dest.to_path_buf();
async move {
let _ = tokio::fs::remove_file(&dest).await;
ApiError::new(
422,
"READ_FAILED",
format!("could not download source: {e}"),
)
}
};
// reqwest byte stream → AsyncBufRead, so the 4-byte magic can be peeked
// without consuming and the whole thing optionally piped through zstd.
let byte_stream = resp
.bytes_stream()
.map(|r| r.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.without_url())));
let mut buf_reader = tokio::io::BufReader::new(tokio_util::io::StreamReader::new(byte_stream));
let is_zstd = {
let head = buf_reader.fill_buf().await.map_err(|e| {
ApiError::new(
422,
"READ_FAILED",
format!("could not download source: {e}"),
)
})?;
decode_zstd && head.len() >= 4 && head[0..4] == [0x28, 0xB5, 0x2F, 0xFD]
};
// multiple_members: tolerate a source written as concatenated zstd frames.
let mut input: std::pin::Pin<Box<dyn tokio::io::AsyncRead + Send>> = if is_zstd {
let mut dec = ZstdDecoder::new(buf_reader);
dec.multiple_members(true);
Box::pin(dec)
} else {
Box::pin(buf_reader)
};
let mut file = tokio::fs::File::create(dest).await.map_err(|e| {
ApiError::new(
500,
"READ_FAILED",
format!("could not write temp file: {e}"),
)
})?;
let mut hasher = xxhash_rust::xxh3::Xxh3::new();
let mut written = 0usize;
let mut buf = vec![0u8; 256 * 1024];
loop {
// A decode error (corrupt/truncated zstd) surfaces here as an io::Error —
// fail loud rather than store a partial file.
let n = match input.read(&mut buf).await {
Ok(0) => break,
Ok(n) => n,
Err(e) => return Err(read_err(e, dest).await),
};
written += n;
if limit > 0 && written > limit {
let _ = tokio::fs::remove_file(dest).await;
return Err(too_large());
}
if let Some(p) = progress {
p.done
.store(written as u64, std::sync::atomic::Ordering::Relaxed);
}
hasher.update(&buf[..n]);
if let Err(e) = file.write_all(&buf[..n]).await {
let _ = tokio::fs::remove_file(dest).await;
return Err(ApiError::new(
500,
"READ_FAILED",
format!("could not write temp file: {e}"),
));
}
}
file.flush().await.ok();
Ok(hasher.digest128())
}
/// A downloaded source's size, for the memory estimate. 0 if it cannot be read.
pub async fn file_len(path: &std::path::Path) -> u64 {
tokio::fs::metadata(path).await.map_or(0, |m| m.len())
}
/// A file's bytes, memory-mapped: the page cache backs them, so uploading a
/// cached artifact does not copy it onto the heap.
pub fn map_file(path: &std::path::Path) -> std::io::Result<bytes::Bytes> {
let file = std::fs::File::open(path)?;
if file.metadata()?.len() == 0 {
return Ok(bytes::Bytes::new());
}
// SAFETY: these files are written once by this service and then only read
// or unlinked; an unlinked file stays mapped until the map is dropped.
let map = unsafe { memmap2::Mmap::map(&file) }?;
Ok(bytes::Bytes::from_owner(map))
}
pub async fn upload(
url: &str,
body: impl Into<reqwest::Body>,
content_type: &str,
) -> Result<(), ApiError> {
let put = put(url, body.into(), content_type);
telemetry::outbound("upload artifact", url, unless_canceled(put)).await
}
async fn put(url: &str, body: reqwest::Body, content_type: &str) -> Result<(), ApiError> {
let deadline = upload_deadline(body.as_bytes().map(|b| b.len() as u64));
let resp = client()
.put(url)
.timeout(deadline)
.header("Content-Type", content_type)
.header("x-upsert", "true") // retried jobs re-upload to the same path
.body(body)
.send()
.await
.map_err(|e| {
ApiError::new(
502,
"UPLOAD_FAILED",
format!("could not upload artifact: {}", e.without_url()),
)
})?;
if !resp.status().is_success() {
let status = resp.status();
let detail = resp.text().await.unwrap_or_default();
let detail: String = detail.chars().take(200).collect();
return Err(ApiError::new(
502,
"UPLOAD_FAILED",
format!("could not upload artifact: {status} {detail}"),
));
}
Ok(())
}
/// POST a JSON body (the completion-callback delivery). Short timeout; the
/// caller owns retries.
pub async fn post_json(url: &str, body: &serde_json::Value) -> Result<(), ApiError> {
telemetry::outbound("callback", url, post(url, body)).await
}
async fn post(url: &str, body: &serde_json::Value) -> Result<(), ApiError> {
let resp = client()
.post(url)
.header("Content-Type", "application/json")
.body(body.to_string())
.timeout(std::time::Duration::from_secs(10))
.send()
.await
.map_err(|e| {
ApiError::new(
502,
"CALLBACK_FAILED",
format!("callback POST failed: {}", e.without_url()),
)
})?;
if !resp.status().is_success() {
return Err(ApiError::new(
502,
"CALLBACK_FAILED",
format!("callback POST returned {}", resp.status()),
));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn a_failed_request_never_echoes_its_signed_url() {
// Nothing listens on port 1: the connect fails, and reqwest's error
// would normally quote the whole URL, token included.
let url = "http://127.0.0.1:1/object/sign/m.step?token=secret-token";
let dest = temp_path("step");
let download = download_with(url, &dest, None, &DownloadOpts::default()).await.err().unwrap();
let upload = upload(url, bytes::Bytes::new(), "application/step").await.err().unwrap();
let callback = post_json(url, &serde_json::json!({})).await.err().unwrap();
for message in [download.message, upload.message, callback.message] {
assert!(!message.contains("secret-token"), "{message}");
assert!(!message.contains("/object/sign"), "{message}");
}
}
#[tokio::test]
async fn a_redirect_is_not_followed() {
let app = axum::Router::new()
.route(
"/object",
axum::routing::get(|| async {
(axum::http::StatusCode::FOUND, [("location", "/elsewhere")])
}),
)
.route("/elsewhere", axum::routing::get(|| async { "unchecked host" }));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let dest = temp_path("step");
let err = download_with(&format!("http://{addr}/object"), &dest, None, &DownloadOpts::default())
.await
.err()
.unwrap();
assert!(err.message.contains("302"), "{}", err.message);
}
/// A CAD system's download stand-in: `/export` needs the bearer and
/// redirects to `/blob` on the SAME host; `/elsewhere` redirects to a
/// second listener, which records whether an Authorization header came.
async fn cad_system(body: &'static [u8]) -> (String, std::sync::Arc<std::sync::atomic::AtomicBool>) {
use axum::http::{HeaderMap, StatusCode};
use std::sync::atomic::{AtomicBool, Ordering};
let leaked = std::sync::Arc::new(AtomicBool::new(false));
let seen = leaked.clone();
let other = axum::Router::new().route(
"/blob",
axum::routing::get(move |headers: HeaderMap| {
let seen = seen.clone();
async move {
seen.store(headers.contains_key("authorization"), Ordering::SeqCst);
body
}
}),
);
let other_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let other_addr = other_listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(other_listener, other).await.unwrap() });
let app = axum::Router::new()
.route(
"/export",
axum::routing::get(|headers: HeaderMap| async move {
if headers.get("authorization").and_then(|v| v.to_str().ok())
== Some("Bearer onshape-token")
{
(StatusCode::FOUND, [("location", "/blob")]).into_response()
} else {
StatusCode::UNAUTHORIZED.into_response()
}
}),
)
.route("/blob", axum::routing::get(move || async move { body }))
.route(
"/elsewhere",
axum::routing::get(move || async move {
(StatusCode::FOUND, [("location", format!("http://{other_addr}/blob"))])
}),
);
use axum::response::IntoResponse;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
(format!("http://{addr}"), leaked)
}
#[tokio::test]
async fn a_host_resolving_only_to_private_addresses_is_refused() {
let err = public_addrs("localhost").await.err().unwrap();
assert!(err.to_string().contains("no public address"), "{err}");
}
#[tokio::test]
async fn a_canceled_job_stops_its_download_at_once() {
// A source that sends headers and then nothing: the download would sit
// in its 60 s read timeout.
let app = axum::Router::new().route(
"/slow",
axum::routing::get(|| async {
let stream = futures_util::stream::pending::<Result<bytes::Bytes, std::io::Error>>();
axum::body::Body::from_stream(stream)
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let dest = temp_path("step");
let url = format!("http://{addr}/slow");
let job = crate::jobs::track("cancel-download", async move {
let token = crate::jobs::current_cancel().unwrap();
tokio::spawn(async move {
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
token.cancel();
});
download_with(&url, &dest, None, &DownloadOpts::default()).await
});
let err = tokio::time::timeout(std::time::Duration::from_secs(5), job)
.await
.expect("the download stopped")
.err()
.unwrap();
assert_eq!(err.code, "canceled");
}
fn fetch_opts(auth: Option<&str>, max_bytes: Option<u64>) -> DownloadOpts<'_> {
// The stand-ins are plain http on 127.0.0.1, which `validate_url` (and
// so the redirect policy) allows only in dev mode, as in local dev.
std::env::set_var("ASSEMBLER_DEV_MODE", "true");
DownloadOpts { authorization: auth, max_bytes, passthrough: true }
}
#[tokio::test]
async fn a_fetch_sends_its_bearer_and_writes_the_bytes_unchanged() {
// zstd magic first: a passthrough must not decode it.
let body: &'static [u8] = &[0x28, 0xB5, 0x2F, 0xFD, 1, 2, 3];
let (base, _) = cad_system(body).await;
let dest = temp_path("fetch");
let hash = download_with(&format!("{base}/export"), &dest, None, &fetch_opts(Some("Bearer onshape-token"), None))
.await
.unwrap();
assert_eq!(std::fs::read(&dest).unwrap(), body);
assert_eq!(hash, xxhash_rust::xxh3::xxh3_128(body));
let _ = std::fs::remove_file(&dest);
let refused = download_with(&format!("{base}/export"), &dest, None, &fetch_opts(None, None))
.await
.err()
.unwrap();
assert!(refused.message.contains("401"), "{}", refused.message);
}
#[tokio::test]
async fn a_fetch_redirected_to_another_host_does_not_carry_the_bearer() {
let (base, leaked) = cad_system(b"ISO-10303-21;").await;
let dest = temp_path("fetch");
download_with(&format!("{base}/elsewhere"), &dest, None, &fetch_opts(Some("Bearer onshape-token"), None))
.await
.unwrap();
let _ = std::fs::remove_file(&dest);
assert!(!leaked.load(std::sync::atomic::Ordering::SeqCst));
}
#[tokio::test]
async fn a_fetch_past_its_cap_is_refused() {
let (base, _) = cad_system(b"ISO-10303-21; far too long").await;
let dest = temp_path("fetch");
let err = download_with(&format!("{base}/export"), &dest, None, &fetch_opts(Some("Bearer onshape-token"), Some(4)))
.await
.err()
.unwrap();
assert_eq!((err.status, err.code.as_str()), (413, "LIMIT_EXCEEDED"));
assert!(!dest.exists());
}
#[test]
fn stale_sources_are_cleared_and_nothing_else() {
let dir = tempfile::tempdir().unwrap();
let stale = dir.path().join("geometry-1-0-123.step");
let other = dir.path().join("asm-cache");
std::fs::write(&stale, b"x").unwrap();
std::fs::create_dir(&other).unwrap();
clear_sources_in(dir.path());
assert!(!stale.exists());
assert!(other.exists());
}
}