Open-source manufacturing ERP, MES and QMS. Quoting, MRP, inventory, shop floor, quality and lot/serial traceability on one Postgres schema, with a REST API and MCP server. Self-host or use Carbon Cloud.
TypeScript ai-agentsbill-of-materialserpinventory-managementleanmanufacturingmanufacturing-execution-systemmcp-servermesmrpopen-source-erppostgresqlproduction-planningqmsquality-management-systemreact-routerself-hostedsupabasetraceabilitytypescript
// SPDX-License-Identifier: AGPL-3.0-only
// Carbon (github.com/crbnos/carbon). Modified or adapted versions of this file,
// including ports, remain AGPLv3; serving them over a network requires releasing their source.
//! Async job store shared by every action (`request::ACTIONS`).
//! One lifecycle for all actions: create -> compute -> upload to the submit-time
//! URLs -> completion callback; GET /v1/jobs/{id}?wait= serves status.
//!
//! Redis-backed, and Redis is REQUIRED (`REDIS_URL`): status +
//! pointers live in Redis, so a restart, a sibling replica, or a different
//! Lambda invocation can still answer the poll — the service is stateless.
//! Boot fails loudly when Redis is missing/unreachable; a silent in-memory
//! fallback would strand cross-instance polls (Lambda dispatch depends on
//! shared state) and hide the misconfiguration.
//!
//! Artifacts are PUT to the caller's signed URLs straight from memory; only a
//! `{result, stats}` POINTER is stored. Artifact bytes never go to Redis: an
//! upload that still fails after its retries fails the job with
//! `UPLOAD_FAILED`, and the caller resubmits with fresh URLs.
use crate::{cache::CODE_VERSION, config, http};
use bytes::Bytes;
use dashmap::DashMap;
use redis::AsyncCommands;
use serde_json::{json, Value};
use std::collections::HashMap;
use std::future::Future;
use std::sync::{Arc, LazyLock};
use std::time::{Duration, Instant};
use tokio::sync::Notify;
use tokio_util::sync::CancellationToken;
/// Waits between upload attempts: four tries in all, about seven seconds.
const UPLOAD_RETRY_DELAYS: [Duration; 3] = [
Duration::from_secs(1),
Duration::from_secs(2),
Duration::from_secs(4),
];
/// One job's state: a Redis HASH (one field each), so a status change can be
/// made conditional on the current status in one script without decoding the
/// rest. `result`/`stats` hold the completion POINTER (paths + counts) — never
/// the artifact bytes.
#[derive(Clone)]
struct JobRecord {
action: String, // one of `request::ACTIONS`
status: String, // pending | starting | running | uploading | done | error | canceled
result: Option<Value>,
stats: Option<Value>,
error: Option<Value>, // { code, message }
/// Signed PUT URLs handed over AT SUBMIT (name → URL), one per output.
/// Never rendered; cleared once the job ends (they are bearer tokens).
/// `callback_url` stays: the Lambda worker re-sends the callback after the
/// job ends, and it only lets its holder report this one job.
upload_urls: Option<HashMap<String, String>>,
/// Completion webhook minted by the caller at submit: the terminal job
/// envelope is POSTed here (fire-with-retries) so the caller's workflow
/// wakes on an event instead of polling. Never rendered.
callback_url: Option<String>,
}
impl JobRecord {
fn from_fields(mut fields: HashMap<String, String>) -> Option<Self> {
fn json<T: serde::de::DeserializeOwned>(v: Option<String>) -> Option<T> {
v.and_then(|s| serde_json::from_str(&s).ok())
}
Some(JobRecord {
action: fields.remove("action")?,
status: fields.remove("status")?,
result: json(fields.remove("result")),
stats: json(fields.remove("stats")),
error: json(fields.remove("error")),
upload_urls: json(fields.remove("upload_urls")),
callback_url: fields.remove("callback_url"),
})
}
}
/// Change a job only while it is still active (the one status check and the
/// write are a single atomic script, so a finish can never overwrite a cancel
/// that landed meanwhile). ARGV: ttl, n, then n field/value pairs to set, then
/// fields to delete. Returns 1 when it changed the job.
static TRANSITION: LazyLock<redis::Script> = LazyLock::new(|| {
redis::Script::new(
r"
local s = redis.call('HGET', KEYS[1], 'status')
if s ~= 'pending' and s ~= 'starting' and s ~= 'running' and s ~= 'uploading' then
return 0
end
local n = tonumber(ARGV[2])
for i = 0, n - 1 do
redis.call('HSET', KEYS[1], ARGV[3 + 2 * i], ARGV[4 + 2 * i])
end
for i = 3 + 2 * n, #ARGV do
redis.call('HDEL', KEYS[1], ARGV[i])
end
redis.call('EXPIRE', KEYS[1], ARGV[1])
return 1
",
)
});
/// How long a running job's lease lives, and how often its worker renews it.
/// A poll that finds an active job with no lease ends it as `WORKER_LOST`: the
/// process running it died (a restart, an OOM kill, a Lambda timeout).
const LEASE_TTL_SECS: u64 = 30;
const LEASE_RENEW_EVERY: Duration = Duration::from_secs(10);
/// The jobs this process is running, with each one's cancel token.
static RUNNING: LazyLock<DashMap<String, CancellationToken>> = LazyLock::new(DashMap::new);
tokio::task_local! {
/// The running job's cancel token, for the downloads and uploads it makes.
static JOB_CANCEL: CancellationToken;
}
/// Run `job` as job `id` of this process: listed for the lease heartbeat, and
/// with a cancel token its transfers watch (`current_cancel`).
pub fn track<F: Future>(id: &str, job: F) -> impl Future<Output = F::Output> {
struct Untrack(String);
impl Drop for Untrack {
fn drop(&mut self) {
RUNNING.remove(&self.0);
}
}
let token = CancellationToken::new();
RUNNING.insert(id.to_string(), token.clone());
let untrack = Untrack(id.to_string());
JOB_CANCEL.scope(token, async move {
let _untrack = untrack;
job.await
})
}
/// The cancel token of the job this task is running, if it is one.
pub fn current_cancel() -> Option<CancellationToken> {
JOB_CANCEL.try_with(CancellationToken::clone).ok()
}
/// One artifact to upload (name → bytes + content type). `Bytes`, so handing it
/// to an upload or a retry is a refcount, never a copy.
#[derive(Clone)]
pub struct Output {
pub name: String,
pub content_type: String,
pub bytes: Bytes,
}
/// The terminal pointer to publish once every output is uploaded.
#[derive(Clone)]
pub struct Done {
pub result: Value,
pub stats: Value,
}
/// Shared job + content-hash result store. Cloned into every handler via
/// `AppState`; inner state is `Arc`/manager-cloned so clones share one store.
#[derive(Clone)]
pub struct JobStore {
conn: redis::aio::ConnectionManager,
/// Per-job in-process wakeups so a same-replica long-poll returns the instant
/// the worker finishes (cross-replica completion caught by the Redis re-check).
notifiers: Notifiers,
}
/// One wakeup per job a long-poll is waiting on. An entry lives only while a
/// poll holds a [`Watch`] on it, so the map does not grow with every job the
/// process has ever been asked about.
#[derive(Clone, Default)]
struct Notifiers(Arc<DashMap<String, Arc<Notify>>>);
struct Watch {
notifiers: Notifiers,
id: String,
notify: Arc<Notify>,
}
impl Notifiers {
fn watch(&self, id: &str) -> Watch {
let notify = self
.0
.entry(id.to_string())
.or_insert_with(|| Arc::new(Notify::new()))
.clone();
Watch {
notifiers: self.clone(),
id: id.to_string(),
notify,
}
}
fn wake(&self, id: &str) {
if let Some(n) = self.0.get(id) {
n.notify_waiters();
}
}
}
impl Drop for Watch {
fn drop(&mut self) {
// The map's handle and this one: no other poll is waiting on the job.
self.notifiers
.0
.remove_if(&self.id, |_, notify| Arc::strong_count(notify) <= 2);
}
}
impl JobStore {
/// Build from env. Redis is REQUIRED — a missing or unreachable
/// `REDIS_URL` refuses to boot (fail loud; a memory fallback would
/// strand cross-instance polls and hide the misconfiguration).
pub async fn from_env() -> Self {
let Some(url) = config::redis_url() else {
eprintln!(
"assembler: Redis is required (set REDIS_URL); refusing to start"
);
std::process::exit(1);
};
match connect(&url).await {
Ok(conn) => {
eprintln!("assembler: redis job store connected");
JobStore {
conn,
notifiers: Notifiers::default(),
}
}
Err(e) => {
eprintln!("assembler: REDIS_URL unreachable ({e}); refusing to start");
std::process::exit(1);
}
}
}
/// Whether Redis answers, for the health check. Bounded, so a connection
/// that hangs fails the check instead of hanging it.
pub async fn ping(&self) -> bool {
let mut c = self.conn.clone();
let ping = redis::cmd("PING");
let answer = ping.query_async::<()>(&mut c);
match tokio::time::timeout(Duration::from_secs(2), answer).await {
Ok(Ok(())) => true,
Ok(Err(e)) => {
eprintln!("assembler: redis ping failed: {e}");
false
}
Err(_) => {
eprintln!("assembler: redis ping timed out");
false
}
}
}
// --- job status (pointer, not artifact bytes) --------------------------
async fn read(&self, id: &str) -> Option<JobRecord> {
self.try_read(id).await.ok().flatten()
}
/// `Err` when Redis could not be asked, as opposed to `Ok(None)` for a job
/// it does not have.
async fn try_read(&self, id: &str) -> Result<Option<JobRecord>, ()> {
let mut c = self.conn.clone();
match c.hgetall::<_, HashMap<String, String>>(job_key(id)).await {
Ok(fields) if fields.is_empty() => Ok(None),
Ok(fields) => Ok(JobRecord::from_fields(fields)),
Err(e) => {
eprintln!("assembler: redis job read failed: {e}");
Err(())
}
}
}
/// Apply `set` and `delete` to an active job; false when the job is gone
/// or already ended (canceled included), so nothing was changed.
async fn transition(&self, id: &str, set: &[(&str, String)], delete: &[&str]) -> bool {
let mut invocation = TRANSITION.key(job_key(id));
invocation.arg(config::job_ttl_secs()).arg(set.len());
for (field, value) in set {
invocation.arg(*field).arg(value);
}
for field in delete {
invocation.arg(*field);
}
let mut c = self.conn.clone();
match invocation.invoke_async::<i64>(&mut c).await {
Ok(changed) => changed == 1,
Err(e) => {
eprintln!("assembler: redis job transition failed: {e}");
false
}
}
}
/// The EXTERNAL status of an active job (queued/running), for the create
/// handler's idempotency-attach 202 response. None if unknown or terminal.
pub async fn existing_active(&self, id: &str) -> Option<String> {
self.internal_active(id)
.await
.map(|s| external_status(&s).to_string())
}
/// The INTERNAL status of an active job (pending/starting/running/uploading).
/// The worker inlet decides on these exact names — the external mapping
/// folds pending/starting->queued and uploading->running.
pub async fn internal_active(&self, id: &str) -> Option<String> {
self.read(id)
.await
.filter(|j| is_active(&j.status))
.map(|j| j.status)
}
pub async fn set_pending(
&self,
id: &str,
action: &str,
upload_urls: HashMap<String, String>,
callback_url: Option<String>,
) {
let mut fields = vec![
("action", action.to_string()),
("status", "pending".to_string()),
];
if !upload_urls.is_empty() {
fields.push(("upload_urls", json!(upload_urls).to_string()));
}
if let Some(url) = callback_url {
fields.push(("callback_url", url));
}
let key = job_key(id);
let mut c = self.conn.clone();
let written = redis::pipe()
.atomic()
.del(&key)
.ignore()
.hset_multiple(&key, &fields)
.ignore()
.expire(&key, config::job_ttl_secs() as i64)
.ignore()
.query_async::<()>(&mut c)
.await;
if let Err(e) = written {
eprintln!("assembler: redis job write failed: {e}");
}
}
/// Move an active job to `status`. A job that is working (not `pending`)
/// holds a lease, renewed by `heartbeat` while this process runs it.
pub async fn set_status(&self, id: &str, status: &str) {
// The lease first: a poll between the two writes would otherwise see a
// working status with no lease and end a live job `WORKER_LOST`. A
// lease left on a job that already ended expires on its own.
if status != "pending" {
self.renew_lease(id).await;
}
self.transition(id, &[("status", status.to_string())], &[]).await;
}
/// Publish a terminal `done` pointer. Does NOT deliver the completion
/// callback; its owner sends it: `finish()`, the plan action's cache-hit
/// publish, and the Lambda worker after `run_to_completion`.
pub async fn set_done(&self, id: &str, done: Done) {
let set = [
("status", "done".to_string()),
("result", done.result.to_string()),
("stats", done.stats.to_string()),
];
if self.transition(id, &set, &["error", "upload_urls"]).await {
self.ended(id).await;
}
self.wake(id);
}
pub async fn set_error(&self, id: &str, code: &str, message: String) {
crate::telemetry::record_error(code, &message);
let set = [
("status", "error".to_string()),
("error", json!({ "code": code, "message": message }).to_string()),
];
if self.transition(id, &set, &["upload_urls"]).await {
self.ended(id).await;
}
self.notify(id);
self.wake(id);
}
/// A job ended: drop its lease and the request a Lambda worker read.
async fn ended(&self, id: &str) {
let mut c = self.conn.clone();
if let Err(e) = c.del::<_, ()>(&[lease_key(id), request_key(id)]).await {
eprintln!("assembler: redis job cleanup failed: {e}");
}
}
/// Deliver the callback in its own task: its retries can take ~36 s, and
/// `set_error`/`cancel` run inside requests that API Gateway cuts at 30 s.
fn notify(&self, id: &str) {
let store = self.clone();
let id = id.to_string();
tokio::spawn(async move { store.send_callback(&id).await });
}
// --- leases: who is running a job --------------------------------------
/// Take a job no live worker holds (a Lambda worker's claim: of two
/// deliveries one wins, and a redelivery after a crash takes over).
pub async fn claim_lease(&self, id: &str) -> bool {
let mut c = self.conn.clone();
let claimed: redis::RedisResult<Option<String>> = redis::cmd("SET")
.arg(lease_key(id))
.arg(1)
.arg("NX")
.arg("EX")
.arg(LEASE_TTL_SECS)
.query_async(&mut c)
.await;
claimed.map_or_else(
|e| {
eprintln!("assembler: redis lease claim failed: {e}");
false
},
|reply| reply.is_some(),
)
}
async fn renew_lease(&self, id: &str) {
let mut c = self.conn.clone();
if let Err(e) = c
.set_ex::<_, _, ()>(lease_key(id), 1, LEASE_TTL_SECS)
.await
{
eprintln!("assembler: redis lease renew failed: {e}");
}
}
/// Whether a live worker holds the job. A Redis that cannot be asked
/// answers yes: a job is never ended over one failed read.
async fn lease_alive(&self, id: &str) -> bool {
let mut c = self.conn.clone();
c.exists::<_, bool>(lease_key(id)).await.unwrap_or(true)
}
/// Every `LEASE_RENEW_EVERY`: renew the lease of each job this process
/// runs, and cancel the token of any that was canceled (from any replica).
/// Runs for the life of the process.
pub async fn heartbeat(&self) {
loop {
tokio::time::sleep(LEASE_RENEW_EVERY).await;
let running: Vec<(String, CancellationToken)> = RUNNING
.iter()
.map(|e| (e.key().clone(), e.value().clone()))
.collect();
if running.is_empty() {
continue;
}
let mut pipe = redis::pipe();
for (id, _) in &running {
pipe.set_ex(lease_key(id), 1, LEASE_TTL_SECS).ignore();
pipe.hget(job_key(id), "status");
}
let mut c = self.conn.clone();
match pipe.query_async::<Vec<Option<String>>>(&mut c).await {
Ok(statuses) => {
for ((_, token), status) in running.iter().zip(statuses) {
if status.as_deref() == Some("canceled") {
token.cancel();
}
}
}
Err(e) => eprintln!("assembler: redis heartbeat failed: {e}"),
}
}
}
// --- Lambda requests ----------------------------------------------------
/// Hold a Lambda job's request body for its worker. The worker spec carries
/// only ids, so signed URLs and caller tokens never ride in the Invoke
/// payload (which AWS keeps for its async retries). Kept until the job
/// ends, so a redelivery after a crashed worker can run it again.
pub async fn put_request(&self, id: &str, body: &Value) -> Result<(), String> {
let mut c = self.conn.clone();
c.set_ex::<_, _, ()>(request_key(id), body.to_string(), REQUEST_TTL_SECS)
.await
.map_err(|e| format!("could not store the request: {e}"))
}
/// A Lambda job's request body; `None` once the job ended or it expired.
pub async fn read_request(&self, id: &str) -> Option<Value> {
let mut c = self.conn.clone();
let raw: Option<String> = c
.get(request_key(id))
.await
.map_err(|e| eprintln!("assembler: redis request read failed: {e}"))
.ok()?;
serde_json::from_str(&raw?).ok()
}
/// Hold the create window for `id`, so two POSTs with one Idempotency-Key
/// cannot both spawn (the active check and the pending write are separate
/// round trips). False while another create holds it. A Redis that cannot
/// be asked does not block the create: its own writes fail just the same.
pub async fn claim_create(&self, id: &str) -> bool {
let mut c = self.conn.clone();
let claimed: redis::RedisResult<Option<String>> = redis::cmd("SET")
.arg(create_key(id))
.arg(1)
.arg("NX")
.arg("EX")
.arg(CREATE_CLAIM_TTL_SECS)
.query_async(&mut c)
.await;
match claimed {
Ok(reply) => reply.is_some(),
Err(e) => {
eprintln!("assembler: redis create claim failed: {e}");
true
}
}
}
pub async fn release_create(&self, id: &str) {
let mut c = self.conn.clone();
if let Err(e) = c.del::<_, ()>(create_key(id)).await {
eprintln!("assembler: redis create release failed: {e}");
}
}
/// Cancel an active job. Its downloads and uploads stop at once on this
/// replica, or within a heartbeat on another; a compute already running
/// finishes and its result is dropped.
pub async fn cancel(&self, id: &str) -> Option<Value> {
if self
.transition(id, &[("status", "canceled".to_string())], &["upload_urls"])
.await
{
if let Some(token) = RUNNING.get(id) {
token.cancel();
}
self.ended(id).await;
// Wake the event-driven waiter immediately (terminal state).
self.notify(id);
self.wake(id);
}
let rec = self.read(id).await?;
Some(Self::render(id, &rec))
}
/// True when the job was canceled or its record is gone — the compute task
/// asks this to abandon a run nobody waits for. A Redis that could not be
/// asked is not a cancel: dropping a finished compute over one failed read
/// would leave the job `running` until its TTL.
pub async fn is_canceled(&self, id: &str) -> bool {
match self.try_read(id).await {
Ok(Some(rec)) => rec.status == "canceled",
Ok(None) => true,
Err(()) => false,
}
}
/// The uniform poll envelope: `{ ok, job: { id, action, status, result?,
/// stats?, error? } }` with the internal status mapped to the public enum.
fn render(id: &str, rec: &JobRecord) -> Value {
let mut job = json!({
"id": id,
"action": rec.action,
"status": external_status(&rec.status),
});
if let Some(r) = &rec.result {
job["result"] = r.clone();
}
if let Some(s) = &rec.stats {
job["stats"] = s.clone();
}
if let Some(e) = &rec.error {
job["error"] = e.clone();
}
json!({ "ok": true, "job": job })
}
/// Long-poll a job to a terminal state. With `max`, holds until terminal or
/// the deadline; without, a single shot. `None` only if the job is unknown.
pub async fn poll(&self, id: &str, max: Option<Duration>) -> Option<Value> {
let deadline = max.map(|m| Instant::now() + m);
let mut watch = None;
loop {
let mut rec = self.read(id).await?;
// Working, but no process holds its lease: whatever ran it died.
if matches!(rec.status.as_str(), "starting" | "running" | "uploading")
&& !self.lease_alive(id).await
{
self.set_error(id, "WORKER_LOST", "the worker running this job stopped".into())
.await;
rec = self.read(id).await?;
}
if matches!(rec.status.as_str(), "done" | "error" | "canceled") {
return Some(Self::render(id, &rec));
}
let Some(dl) = deadline else {
return Some(Self::render(id, &rec)); // single shot
};
let now = Instant::now();
if now >= dl {
return Some(Self::render(id, &rec));
}
let watch = watch.get_or_insert_with(|| self.notifiers.watch(id));
let tick = (dl - now).min(Duration::from_millis(500));
tokio::select! {
_ = watch.notify.notified() => {}
_ = tokio::time::sleep(tick) => {}
}
}
}
pub fn wake(&self, id: &str) {
self.notifiers.wake(id);
}
/// Complete a computed job: upload every output to its submit-time URL,
/// then publish the terminal pointer and deliver the callback. An upload
/// that fails after its retries fails the job with `UPLOAD_FAILED`; the
/// outputs are dropped, never kept for later.
pub async fn finish(
&self,
id: &str,
outputs: Vec<Output>,
done: Done,
cache: Option<(String, u128, u64)>,
) {
// A cancel that landed mid-compute wins: drop the result, upload nothing.
if self.is_canceled(id).await {
eprintln!("[{id}] finished after cancel; result dropped");
return;
}
self.set_status(id, "uploading").await;
let urls = self
.read(id)
.await
.and_then(|r| r.upload_urls)
.unwrap_or_default();
let uploaded = upload_all(id, &outputs, &urls, &UPLOAD_RETRY_DELAYS).await;
// Nothing reads the bytes past this point; don't hold them through
// the callback's retries.
drop(outputs);
match uploaded {
Ok(()) => {
self.publish(id, done, cache).await;
// Terminal — deliver the callback from the action task, which
// owns the process lifetime on the server path (the Lambda
// worker re-sends after run_to_completion regardless).
self.send_callback(id).await;
self.wake(id);
}
Err(message) => self.set_error(id, "UPLOAD_FAILED", message).await,
}
}
/// POST the terminal job envelope to the submit-time callback URL, if any.
/// Bounded retries; failure is logged only — the caller's waitForEvent
/// timeout + fallback poll covers a lost callback.
pub async fn send_callback(&self, id: &str) {
let Some(rec) = self.read(id).await else {
return;
};
let Some(url) = rec.callback_url.clone() else {
return;
};
if !matches!(rec.status.as_str(), "done" | "error" | "canceled") {
return;
}
let body = Self::render(id, &rec);
for attempt in 1..=3u32 {
match http::post_json(&url, &body).await {
Ok(()) => {
eprintln!("[{id}] completion callback delivered");
return;
}
Err(e) => {
eprintln!(
"[{id}] completion callback attempt {attempt} failed: {}",
e.message
);
tokio::time::sleep(Duration::from_secs(attempt as u64)).await;
}
}
}
eprintln!("[{id}] completion callback undelivered; caller falls back to poll");
}
/// Every output is uploaded: record the result pointer and mark the job done.
async fn publish(&self, id: &str, done: Done, cache: Option<(String, u128, u64)>) {
if let Some((model, content, opts)) = cache {
let pointer = json!({ "result": done.result, "stats": done.stats });
self.result_put(&model, content, opts, pointer).await;
}
self.set_done(id, done).await;
}
// --- content-hash result-pointer cache (CODE_VERSION-stamped) ----------
pub async fn result_get(&self, model: &str, content: u128, opts: u64) -> Option<Done> {
let key = result_key(model, content, opts);
let mut c = self.conn.clone();
let raw: Option<Value> = match c.get::<_, Option<String>>(&key).await {
Ok(Some(s)) => serde_json::from_str(&s).ok(),
Ok(None) => None,
Err(e) => {
eprintln!("assembler: redis result read failed: {e}");
None
}
};
raw.map(|v| Done {
result: v["result"].clone(),
stats: v["stats"].clone(),
})
}
async fn result_put(&self, model: &str, content: u128, opts: u64, pointer: Value) {
let key = result_key(model, content, opts);
let Ok(payload) = serde_json::to_string(&pointer) else {
return;
};
let mut c = self.conn.clone();
if let Err(e) = c
.set_ex::<_, _, ()>(&key, payload, config::result_ttl_secs())
.await
{
eprintln!("assembler: redis result write failed: {e}");
}
}
/// Central explicit invalidation: drop every cached result pointer for a
/// model so the next job re-derives. Returns how many entries were cleared.
pub async fn invalidate_model(&self, model: &str) -> usize {
let pattern = format!("asm:result:{}:*", escape_glob(model));
let mut scan = self.conn.clone();
let mut keys: Vec<String> = Vec::new();
match scan.scan_match::<_, String>(&pattern).await {
Ok(mut iter) => {
while let Some(k) = iter.next_item().await {
keys.push(k);
}
}
Err(e) => {
eprintln!("assembler: redis invalidate scan failed: {e}");
return 0;
}
}
if keys.is_empty() {
return 0;
}
let mut c = self.conn.clone();
if let Err(e) = c.del::<_, ()>(&keys).await {
eprintln!("assembler: redis invalidate del failed: {e}");
return 0;
}
keys.len()
}
}
/// The client's defaults suit nobody waiting on a job: no limit on a connect
/// or a reply, and a reconnect backoff that goes from one second straight to a
/// minute or two (its factor of 100 multiplies each delay). After a Redis
/// restart that left every job read and write failing for about two minutes.
/// Bounded waits, and retries that double from one second up to five.
const REDIS_CONNECT_TIMEOUT: Duration = Duration::from_secs(5);
const REDIS_RESPONSE_TIMEOUT: Duration = Duration::from_secs(15);
const REDIS_RETRY_FACTOR: u64 = 2;
const REDIS_RETRY_MAX_DELAY_MS: u64 = 5_000;
async fn connect(url: &str) -> redis::RedisResult<redis::aio::ConnectionManager> {
let client = redis::Client::open(url)?;
let config = redis::aio::ConnectionManagerConfig::new()
.set_connection_timeout(REDIS_CONNECT_TIMEOUT)
.set_response_timeout(REDIS_RESPONSE_TIMEOUT)
.set_factor(REDIS_RETRY_FACTOR)
.set_max_delay(REDIS_RETRY_MAX_DELAY_MS);
let mut conn = client.get_connection_manager_with_config(config).await?;
redis::cmd("PING").query_async::<()>(&mut conn).await?;
Ok(conn)
}
/// Map an internal status to the public job-status enum.
/// `starting`: a worker took the job and is downloading its source or waiting
/// for memory. It is still `queued` to callers (queue time is not work), but a
/// second delivery of the same job sees it as taken. `running` is set only
/// once admission grants the memory.
fn is_active(status: &str) -> bool {
matches!(status, "pending" | "starting" | "running" | "uploading")
}
fn external_status(internal: &str) -> &'static str {
match internal {
"pending" | "starting" => "queued",
"running" | "uploading" => "running",
"done" => "succeeded",
"error" => "failed",
"canceled" => "canceled",
_ => "running",
}
}
/// `asm:job2`: the record became a hash (it was a JSON string under
/// `asm:job`), and a new prefix keeps the two from meeting during a deploy.
fn job_key(id: &str) -> String {
format!("asm:job2:{id}")
}
fn lease_key(id: &str) -> String {
format!("asm:lease:{id}")
}
/// How long a Lambda job's request is kept at most (it is deleted when the job
/// ends): long enough for AWS's async retries to redeliver a crashed job.
const REQUEST_TTL_SECS: u64 = 2 * 60 * 60;
/// How long a create may hold its window; released as soon as it is done.
const CREATE_CLAIM_TTL_SECS: u64 = 30;
fn create_key(id: &str) -> String {
format!("asm:create:{id}")
}
fn request_key(id: &str) -> String {
format!("asm:req:{id}")
}
/// Escape Redis glob metacharacters, so a caller's id matches only itself.
fn escape_glob(s: &str) -> String {
let mut out = String::with_capacity(s.len());
for c in s.chars() {
if matches!(c, '*' | '?' | '[' | ']' | '\\') {
out.push('\\');
}
out.push(c);
}
out
}
fn result_key(model: &str, content: u128, opts: u64) -> String {
format!("asm:result:{model}:{content:032x}:{opts:016x}:v{CODE_VERSION}")
}
/// PUT every output to its URL, retrying each one after every wait in
/// `delays`. An output with no URL, or one still failing after its retries,
/// is the error (the message names the output).
async fn upload_all(
id: &str,
outputs: &[Output],
urls: &HashMap<String, String>,
delays: &[Duration],
) -> Result<(), String> {
for o in outputs {
let Some(url) = urls.get(&o.name) else {
return Err(format!("no upload URL for output '{}'", o.name));
};
let mut waits = delays.iter();
loop {
match http::upload(url, o.bytes.clone(), &o.content_type).await {
Ok(()) => break,
Err(e) => match waits.next() {
Some(wait) => {
eprintln!(
"[{id}] output '{}' upload failed, retrying in {wait:?}: {}",
o.name, e.message
);
tokio::time::sleep(*wait).await;
}
None => {
return Err(format!("output '{}' upload failed: {}", o.name, e.message))
}
},
}
}
}
Ok(())
}
/// Stable hash of an options object. `serde_json` serializes object keys sorted,
/// so the string is deterministic.
pub fn opts_hash(options: &Value) -> u64 {
xxhash_rust::xxh3::xxh3_64(options.to_string().as_bytes())
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicUsize, Ordering};
#[test]
fn a_wakeup_lives_only_while_a_poll_waits_on_it() {
let notifiers = Notifiers::default();
let first = notifiers.watch("job");
let second = notifiers.watch("job");
assert!(Arc::ptr_eq(&first.notify, &second.notify));
drop(first);
assert_eq!(notifiers.0.len(), 1, "the second poll still waits");
drop(second);
assert!(notifiers.0.is_empty());
}
#[test]
fn a_model_id_matches_only_itself_in_a_scan() {
assert_eq!(escape_glob("model-1"), "model-1");
assert_eq!(escape_glob("*"), "\\*");
assert_eq!(escape_glob("a?b[c]\\"), "a\\?b\\[c\\]\\\\");
}
#[test]
fn a_taken_job_is_still_queued_to_callers_and_active() {
// A worker took it but has not been granted memory: callers must not
// start its time budget, and a second delivery must see it as taken.
assert_eq!(external_status("starting"), "queued");
assert!(is_active("starting"));
assert!(!is_active("canceled"));
}
/// A storage stand-in that answers 503 to the first `failures` PUTs and
/// 200 after, counting every attempt.
async fn flaky_storage(failures: usize) -> (String, Arc<AtomicUsize>) {
let attempts = Arc::new(AtomicUsize::new(0));
let seen = attempts.clone();
let app = axum::Router::new().route(
"/upload",
axum::routing::put(move || {
let seen = seen.clone();
async move {
if seen.fetch_add(1, Ordering::SeqCst) < failures {
axum::http::StatusCode::SERVICE_UNAVAILABLE
} else {
axum::http::StatusCode::OK
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
(format!("http://{addr}/upload"), attempts)
}
fn glb() -> Vec<Output> {
vec![Output {
name: "glb".into(),
content_type: "model/gltf-binary".into(),
bytes: Bytes::from_static(b"glTF"),
}]
}
const NO_WAIT: [Duration; 3] = [Duration::ZERO; 3];
/// A store on the Redis in `REDIS_URL` (the local stack's), or `None` to
/// skip: these pin behaviour that only Redis itself can show.
async fn redis_store() -> Option<JobStore> {
let url = std::env::var("REDIS_URL").ok().filter(|u| !u.is_empty())?;
let conn = connect(&url).await.ok()?;
Some(JobStore {
conn,
notifiers: Notifiers::default(),
})
}
fn unique(name: &str) -> String {
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos();
format!("test-{name}-{nanos}")
}
#[tokio::test]
async fn one_create_window_per_job_id() {
let Some(store) = redis_store().await else { return };
let id = unique("claim");
assert!(store.claim_create(&id).await);
assert!(!store.claim_create(&id).await, "a second create waits it out");
store.release_create(&id).await;
assert!(store.claim_create(&id).await);
store.release_create(&id).await;
}
#[tokio::test]
async fn a_lambda_request_is_kept_until_its_job_ends() {
let Some(store) = redis_store().await else { return };
let id = unique("request");
let body = json!({ "source": { "url": "https://storage.test/m.step" } });
store.set_pending(&id, "thumbnail", HashMap::new(), None).await;
store.put_request(&id, &body).await.unwrap();
assert_eq!(store.read_request(&id).await, Some(body.clone()));
assert_eq!(store.read_request(&id).await, Some(body), "a redelivery reads it too");
store.set_error(&id, "thumbnail_failed", "x".into()).await;
assert_eq!(store.read_request(&id).await, None);
}
#[tokio::test]
async fn one_worker_claims_a_job_and_a_crashed_one_releases_it() {
let Some(store) = redis_store().await else { return };
let id = unique("lease");
assert!(store.claim_lease(&id).await);
assert!(!store.claim_lease(&id).await, "a second delivery is a no-op");
let mut c = store.conn.clone();
let _: () = c.del(lease_key(&id)).await.unwrap(); // the lease ran out
assert!(store.claim_lease(&id).await, "a redelivery takes over");
}
#[tokio::test]
async fn a_job_whose_worker_died_ends_as_worker_lost() {
let Some(store) = redis_store().await else { return };
let id = unique("lost");
store.set_pending(&id, "thumbnail", HashMap::new(), None).await;
store.set_status(&id, "running").await;
let job = store.poll(&id, None).await.unwrap();
assert_eq!(job["job"]["status"], "running", "its lease is live");
let mut c = store.conn.clone();
let _: () = c.del(lease_key(&id)).await.unwrap(); // no renewal came
let job = store.poll(&id, None).await.unwrap();
assert_eq!(job["job"]["status"], "failed");
assert_eq!(job["job"]["error"]["code"], "WORKER_LOST");
}
#[tokio::test]
async fn a_finish_never_overwrites_a_cancel() {
let Some(store) = redis_store().await else { return };
let id = unique("cancel");
store.set_pending(&id, "thumbnail", HashMap::new(), None).await;
store.set_status(&id, "running").await;
store.cancel(&id).await;
store.set_done(&id, Done { result: json!({}), stats: json!({}) }).await;
store.set_error(&id, "thumbnail_failed", "late".into()).await;
store.set_status(&id, "uploading").await;
let job = store.poll(&id, None).await.unwrap();
assert_eq!(job["job"]["status"], "canceled");
}
#[tokio::test]
async fn a_cancel_fires_the_running_jobs_token() {
let Some(store) = redis_store().await else { return };
let id = unique("token");
store.set_pending(&id, "thumbnail", HashMap::new(), None).await;
let (fired_tx, fired_rx) = tokio::sync::oneshot::channel();
let job = track(&id, async move {
current_cancel().unwrap().cancelled().await;
let _ = fired_tx.send(());
});
let running = tokio::spawn(job);
tokio::task::yield_now().await;
store.cancel(&id).await;
tokio::time::timeout(Duration::from_secs(1), fired_rx).await.unwrap().unwrap();
running.await.unwrap();
assert!(!RUNNING.contains_key(&id), "an ended job is untracked");
}
#[tokio::test]
async fn a_failed_upload_fails_the_job_and_parks_nothing() {
let Some(store) = redis_store().await else { return };
let (url, _) = flaky_storage(usize::MAX).await;
let id = unique("upload");
store
.set_pending(&id, "thumbnail", HashMap::from([("glb".to_string(), url)]), None)
.await;
let done = Done { result: json!({}), stats: json!({}) };
store.finish(&id, glb(), done, None).await;
let job = store.poll(&id, None).await.unwrap();
assert_eq!(job["job"]["status"], "failed");
assert_eq!(job["job"]["error"]["code"], "UPLOAD_FAILED");
let mut c = store.conn.clone();
let parked: Vec<String> = c.keys(format!("asm:pending:{id}*")).await.unwrap();
assert!(parked.is_empty());
let rec = store.read(&id).await.unwrap();
assert!(rec.upload_urls.is_none(), "an ended job keeps no signed URLs");
}
#[tokio::test]
async fn an_upload_that_fails_twice_then_lands_succeeds() {
let (url, attempts) = flaky_storage(2).await;
let urls = HashMap::from([("glb".to_string(), url)]);
assert_eq!(upload_all("job", &glb(), &urls, &NO_WAIT).await, Ok(()));
assert_eq!(attempts.load(Ordering::SeqCst), 3);
}
#[tokio::test]
async fn an_upload_still_failing_after_its_retries_is_an_error() {
let (url, attempts) = flaky_storage(usize::MAX).await;
let urls = HashMap::from([("glb".to_string(), url)]);
let err = upload_all("job", &glb(), &urls, &NO_WAIT).await.unwrap_err();
assert!(err.starts_with("output 'glb' upload failed"), "{err}");
assert_eq!(attempts.load(Ordering::SeqCst), 4, "one try plus three retries");
}
#[tokio::test]
async fn an_output_without_a_url_is_an_error_not_a_wait() {
let err = upload_all("job", &glb(), &HashMap::new(), &NO_WAIT).await.unwrap_err();
assert_eq!(err, "no upload URL for output 'glb'");
}
}