Python
use crate::Config;
use anyhow::Result;
use axum::{
extract::{DefaultBodyLimit, Path, Query, Request, State},
http::{header, HeaderMap, StatusCode},
middleware::{self, Next},
response::{
sse::{Event, KeepAlive, Sse},
IntoResponse, Response,
},
routing::{get, post},
Extension, Json, Router,
};
use openworker_runtime::runtime::Runtime;
use openworker_store_postgres::auth::{Action, Error as AuthError};
use openworker_store_postgres::{PostgresStore, SessionCreation, SubmissionError, TurnSubmission};
use serde::Deserialize;
use serde_json::{json, Value};
use std::{convert::Infallible, sync::Arc, time::Duration};
use tokio::sync::Semaphore;
use tokio_util::sync::CancellationToken;
#[derive(Clone)]
pub struct App {
pub(crate) recording_readers: Arc<Semaphore>,
pub(crate) safety_readers: Arc<Semaphore>,
recovery_read_only: bool,
pub(crate) store: PostgresStore,
pub(crate) runtime: Arc<Runtime>,
pub(crate) actor: String,
streams: Arc<Semaphore>,
pub(crate) service: Option<crate::service::ServiceConfig>,
pub(crate) config: Arc<Config>,
pub(crate) stop: CancellationToken,
pub(crate) artifacts: Option<openworker_artifacts::Store>,
pub(crate) transfers: Arc<Semaphore>,
}
impl App {
pub fn new(
config: &Config,
runtime: Arc<Runtime>,
store: PostgresStore,
stop: CancellationToken,
) -> Result<Self> {
if config.actor.trim().is_empty() || config.actor.len() > 128 {
anyhow::bail!("Invalid reference principal");
}
if let Some(service) = &config.service {
service.validate()?;
if config.recovery_read_only {
store.verify_service(&service.id, &service.namespace)?;
} else {
store.bind_service(&service.id, &service.namespace)?;
}
}
Ok(Self {
recording_readers: Arc::new(Semaphore::new(4)),
safety_readers: Arc::new(Semaphore::new(8)),
config: Arc::new(config.clone()),
service: config.service.clone(),
recovery_read_only: config.recovery_read_only,
artifacts: config.artifacts.as_ref().map(|a| a.build()).transpose()?,
transfers: Arc::new(Semaphore::new(4)),
store,
runtime,
actor: config.actor.clone(),
streams: Arc::new(Semaphore::new(32)),
stop,
})
}
pub fn router(self) -> Router {
Router::new()
.merge(crate::service::router())
.merge(crate::recording_query::router(""))
.merge(crate::recording_query::router("/v1"))
.route(
"/healthz",
get(|| async { Json(json!({"status":"up","reference":true})) }),
)
.route("/sessions", post(create).get(session_page))
.route("/teams", post(team_create))
.route("/teams/:id", get(team_snapshot))
.route("/teams/:id/commands", post(team_command))
.route("/teams/:id/events", get(team_events))
.route("/teams/:id/events/page", get(team_event_page))
.route("/readyz", get(ready))
.route("/auth/me", get(crate::authorization::me))
.route(
"/credentials",
get(crate::authorization::list).post(crate::authorization::issue),
)
.route(
"/credentials/:id/revoke",
post(crate::authorization::revoke),
)
.route("/security/audit", get(crate::authorization::audit))
.route("/sessions/:sid", get(snapshot))
.route("/sessions/:sid/events", get(events))
.route("/sessions/:sid/turns", post(submit))
.route("/sessions/:sid/turns/:id", get(receipt))
.route("/sessions/:sid/controls", post(control))
.route("/sessions/:sid/controls/:id", get(control_receipt))
.route(
"/sessions/:sid/artifacts",
get(crate::artifacts::list).post(crate::artifacts::register),
)
.route(
"/sessions/:sid/artifacts/:id",
get(crate::artifacts::metadata),
)
.route(
"/sessions/:sid/artifacts/:id/reconcile",
post(crate::artifacts::reconcile),
)
.route(
"/sessions/:sid/artifacts/:id/content",
get(crate::artifacts::download)
.put(crate::artifacts::upload)
.layer(DefaultBodyLimit::max(
openworker_artifacts::MAX_SIZE as usize + 1,
)),
)
.layer(DefaultBodyLimit::max(160_000))
.layer(middleware::from_fn_with_state(self.clone(), authenticate))
.with_state(self)
}
}
async fn authenticate(State(app): State<App>, request: Request, next: Next) -> Response {
let recording_route =
crate::recording_query::authority(request.method().as_str(), request.uri().path())
|| request.uri().path().starts_with("/v1/compliance/");
let span = app
.runtime
.telemetry
.span(openworker_telemetry::Operation::Api, None, None);
let service = request
.uri()
.path()
.starts_with("/v1/")
.then(|| app.service.clone())
.flatten();
let mut result =
openworker_telemetry::scope(span.context(), authenticate_inner(app, request, next)).await;
// Include identity and a fresh trace ID on errors too. This ID is never an
// admission/deduplication key and is not supplied by an untrusted caller.
if let Some(config) = service {
result.headers_mut().insert(
"x-openworker-service-id",
config.id.parse().expect("validated"),
);
result.headers_mut().insert(
"x-openworker-namespace",
config.namespace.parse().expect("validated"),
);
result.headers_mut().insert(
"x-request-id",
openworker_runtime::id("req").parse().unwrap(),
);
}
if recording_route {
result.headers_mut().insert(
header::CACHE_CONTROL,
axum::http::HeaderValue::from_static("no-store"),
);
}
span.finish(if result.status().is_success() {
openworker_telemetry::Outcome::Ok
} else {
openworker_telemetry::Outcome::Error
});
result
}
async fn authenticate_inner(mut app: App, mut request: Request, next: Next) -> Response {
let supplied = request
.headers()
.get(header::AUTHORIZATION)
.and_then(|h| h.to_str().ok())
.and_then(|s| s.strip_prefix("Bearer "))
.unwrap_or("");
app.store = match app.store.authorized(supplied) {
Ok(s) => s,
Err(_) => return ApiError(AuthError::Unauthorized.into()).into_response(),
};
let p = match app.store.authorize(Action::Identify, None, None) {
Ok(p) => p,
Err(e) => return ApiError(e).into_response(),
};
app.actor = p.principal;
if request.headers().contains_key(header::ORIGIN) {
return (
StatusCode::FORBIDDEN,
Json(json!({"error":"browser_origin_not_enabled"})),
)
.into_response();
}
if app.recovery_read_only
&& !matches!(
*request.method(),
axum::http::Method::GET | axum::http::Method::HEAD
)
{
return (
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({"error":"recovery_read_only"})),
)
.into_response();
}
if app.stop.is_cancelled() {
return (
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({"error":"shutting_down"})),
)
.into_response();
}
if app
.service
.as_ref()
.is_some_and(|s| !s.public_path(request.uri().path()))
{
return StatusCode::NOT_FOUND.into_response();
}
let service_request = request.uri().path().starts_with("/v1/");
if service_request {
let Some(service) = &app.service else {
return StatusCode::NOT_FOUND.into_response();
};
if request
.headers()
.get("x-openworker-service-id")
.is_some_and(|v| v.to_str().ok() != Some(service.id.as_str()))
{
return ApiError(AuthError::Forbidden.into()).into_response();
}
if request
.headers()
.get("x-openworker-namespace")
.and_then(|h| h.to_str().ok())
!= Some(service.namespace.as_str())
{
return ApiError(AuthError::Forbidden.into()).into_response();
}
}
let Some(action) = crate::authorization::route(request.method().as_str(), request.uri().path())
else {
return ApiError(AuthError::Forbidden.into()).into_response();
};
if let Err(e) = app.store.authorize(action, None, None) {
let _ = app.store.auth_denied(action);
return ApiError(e).into_response();
}
request.extensions_mut().insert(app.clone());
let response = next.run(request).await;
if matches!(
response.status(),
StatusCode::FORBIDDEN | StatusCode::UNAUTHORIZED
) {
let _ = app.store.auth_denied(action);
}
response
}
async fn ready(Extension(app): Extension<App>) -> Result<Json<Value>, ApiError> {
app.store.ready()?;
Ok(Json(
json!({"status":"ready","recovery_read_only":app.recovery_read_only,"telemetry_enabled":app.runtime.telemetry.enabled(),"telemetry":app.runtime.telemetry.snapshot(),"recording":app.config.recording_stats.as_ref().map(|s|s.snapshot())}),
))
}
pub(crate) struct ApiError(pub(crate) anyhow::Error);
impl From<anyhow::Error> for ApiError {
fn from(e: anyhow::Error) -> Self {
Self(e)
}
}
impl IntoResponse for ApiError {
fn into_response(self) -> Response {
if self
.0
.is::<openworker_runtime::attachments::InspectionUnsupported>()
{
return (
StatusCode::UNPROCESSABLE_ENTITY,
Json(json!({"error":"attachment_inspection_unsupported"})),
)
.into_response();
}
if let Some(error) = self.0.downcast_ref::<crate::service_safety::InputError>() {
return match error {
crate::service_safety::InputError::Blocked => (
StatusCode::UNPROCESSABLE_ENTITY,
Json(json!({"error":"policy_blocked"})),
)
.into_response(),
crate::service_safety::InputError::Unavailable => (
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({"error":"guardrail_unavailable"})),
)
.into_response(),
};
}
if self
.0
.downcast_ref::<crate::git_access::Unavailable>()
.is_some()
{
return (
StatusCode::BAD_REQUEST,
Json(json!({"error":"git_grant_unavailable"})),
)
.into_response();
}
if let Some(error) = self.0.downcast_ref::<crate::recording_query::Error>() {
use crate::recording_query::Error as E;
let (status, code) = match error {
E::Invalid => (StatusCode::BAD_REQUEST, "invalid_recording_query"),
E::Unavailable => (StatusCode::SERVICE_UNAVAILABLE, "recording_unavailable"),
E::Busy => (StatusCode::TOO_MANY_REQUESTS, "recording_query_busy"),
};
return (status, Json(json!({"error":code}))).into_response();
}
if let Some(error) = self
.0
.downcast_ref::<openworker_store_postgres::teams::Error>()
{
use openworker_store_postgres::teams::Error as E;
let status = match error {
E::Invalid => StatusCode::BAD_REQUEST,
E::Conflict => StatusCode::CONFLICT,
E::NotFound => StatusCode::NOT_FOUND,
E::Limit => StatusCode::TOO_MANY_REQUESTS,
};
return (status, Json(json!({"error":error.to_string()}))).into_response();
}
if let Some(error) = self.0.downcast_ref::<AuthError>() {
let status = match error {
AuthError::Unauthorized => StatusCode::UNAUTHORIZED,
AuthError::Forbidden => StatusCode::FORBIDDEN,
AuthError::Invalid => StatusCode::BAD_REQUEST,
AuthError::Conflict => StatusCode::CONFLICT,
AuthError::Unconfigured => StatusCode::SERVICE_UNAVAILABLE,
};
return (status, Json(json!({"error":error.to_string()}))).into_response();
}
if let Some(error) = self.0.downcast_ref::<openworker_artifacts::Error>() {
use openworker_artifacts::Error as E;
let status = match error {
E::Invalid => StatusCode::BAD_REQUEST,
E::NotFound => StatusCode::NOT_FOUND,
E::Conflict | E::NotReady => StatusCode::CONFLICT,
E::TooLarge => StatusCode::PAYLOAD_TOO_LARGE,
E::Cancelled => StatusCode::REQUEST_TIMEOUT,
_ => StatusCode::SERVICE_UNAVAILABLE,
};
return (status, Json(json!({"error":error.to_string()}))).into_response();
}
let (status, code) = match self.0.downcast_ref::<SubmissionError>() {
Some(SubmissionError::InvalidRequest) => (StatusCode::BAD_REQUEST, "invalid_request"),
Some(SubmissionError::NotFound) => (StatusCode::NOT_FOUND, "not_found"),
Some(SubmissionError::Conflict) => (StatusCode::CONFLICT, "idempotency_conflict"),
Some(SubmissionError::QueueFull) => (StatusCode::TOO_MANY_REQUESTS, "queue_full"),
Some(SubmissionError::Busy) => (StatusCode::CONFLICT, "session_busy"),
Some(SubmissionError::Closed) => (StatusCode::CONFLICT, "session_closed"),
Some(SubmissionError::ControlConflict) => {
(StatusCode::CONFLICT, "control_target_conflict")
}
Some(SubmissionError::ControlsPending) => (StatusCode::CONFLICT, "controls_pending"),
Some(SubmissionError::ProviderBlocked) => (StatusCode::CONFLICT, "provider_blocked"),
_ => (
StatusCode::SERVICE_UNAVAILABLE,
"operation_failed_reconcile_receipt",
),
};
(status, Json(json!({"error":code}))).into_response()
}
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Create {
request_id: String,
parameters: Value,
}
async fn create(
Extension(app): Extension<App>,
Json(body): Json<Create>,
) -> Result<Response, ApiError> {
let request = SessionCreation {
request_id: body.request_id,
actor: app.actor,
parameters: body.parameters,
};
if let Some(receipt) = app.store.creation(request.clone())? {
return Ok((StatusCode::OK, Json(receipt)).into_response());
}
let candidate = app
.runtime
.prepare_session(request.parameters.clone())
.map_err(|_| ApiError(SubmissionError::InvalidRequest.into()))?;
let receipt = app.store.create_session(request, candidate)?;
Ok((
if receipt.duplicate {
StatusCode::OK
} else {
StatusCode::CREATED
},
Json(receipt),
)
.into_response())
}
async fn snapshot(
Extension(app): Extension<App>,
Path(sid): Path<String>,
) -> Result<Json<Value>, ApiError> {
Ok(Json(app.store.read_snapshot(&sid)?))
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct SessionPage {
after: Option<String>,
limit: Option<u16>,
}
pub(crate) async fn session_page(
Extension(app): Extension<App>,
Query(q): Query<SessionPage>,
) -> Result<Json<Value>, ApiError> {
Ok(Json(
app.store.session_page(q.after, q.limit.unwrap_or(50))?,
))
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Turn {
submission_id: String,
text: String,
#[serde(default)]
limits: Option<openworker_runtime::budget::Limits>,
}
async fn submit(
Extension(app): Extension<App>,
Path(sid): Path<String>,
Json(body): Json<Turn>,
) -> Result<Response, ApiError> {
let text =
crate::service_safety::inspect(&app, Some(&sid), &body.submission_id, &body.text).await?;
Ok((
StatusCode::ACCEPTED,
Json(app.store.submit_turn(TurnSubmission {
session_id: sid,
submission_id: body.submission_id,
text,
limits: body.limits,
})?),
)
.into_response())
}
async fn receipt(
Extension(app): Extension<App>,
Path((sid, id)): Path<(String, String)>,
) -> Result<Response, ApiError> {
Ok(Json(app.store.submission(&sid, &id)?).into_response())
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Control {
request_id: String,
turn_id: String,
action: openworker_runtime::control::Action,
}
async fn control(
Extension(app): Extension<App>,
Path(sid): Path<String>,
Json(body): Json<Control>,
) -> Result<Response, ApiError> {
let action =
crate::service_safety::inspect_control(&app, &sid, &body.request_id, body.action).await?;
Ok((
StatusCode::ACCEPTED,
Json(
app.store
.submit_control(openworker_runtime::control::Command {
session_id: sid,
request_id: body.request_id,
turn_id: body.turn_id,
actor: app.actor,
action,
})?,
),
)
.into_response())
}
async fn control_receipt(
Extension(app): Extension<App>,
Path((sid, id)): Path<(String, String)>,
) -> Result<Response, ApiError> {
Ok(Json(app.store.control_receipt(&sid, &id)?).into_response())
}
#[derive(Default, Deserialize)]
pub(crate) struct Cursor {
after: Option<i64>,
}
async fn team_create(
Extension(app): Extension<App>,
Json(body): Json<openworker_store_postgres::teams::Create>,
) -> Result<Response, ApiError> {
Ok((StatusCode::CREATED, Json(app.store.create_team(body)?)).into_response())
}
async fn team_snapshot(
Extension(app): Extension<App>,
Path(id): Path<String>,
) -> Result<Json<Value>, ApiError> {
Ok(Json(app.store.team_snapshot(&id)?))
}
async fn team_command(
Extension(app): Extension<App>,
Path(id): Path<String>,
Json(body): Json<openworker_store_postgres::teams::Command>,
) -> Result<Json<Value>, ApiError> {
Ok(Json(app.store.team_command(&id, body)?))
}
async fn team_event_page(
Extension(app): Extension<App>,
Path(id): Path<String>,
Query(q): Query<Cursor>,
) -> Result<Json<Value>, ApiError> {
Ok(Json(app.store.team_events(&id, q.after.unwrap_or(0))?))
}
async fn team_events(
Extension(app): Extension<App>,
Path(id): Path<String>,
Query(q): Query<Cursor>,
headers: HeaderMap,
) -> Result<Response, ApiError> {
let header = headers
.get("last-event-id")
.map(|v| {
v.to_str()
.ok()
.and_then(|s| s.parse::<i64>().ok())
.ok_or_else(|| ApiError(SubmissionError::InvalidRequest.into()))
})
.transpose()?;
let mut cursor = header.or(q.after).unwrap_or(0);
let initial = app.store.team_events(&id, cursor)?;
let permit = app
.streams
.clone()
.try_acquire_owned()
.map_err(|_| ApiError(SubmissionError::QueueFull.into()))?;
let stream = async_stream::stream! {
let _permit=permit;
let mut batch=initial;
loop {
for event in batch["events"].as_array().expect("team event array") {
cursor=event["cursor"].as_i64().expect("team cursor");
yield Ok::<_,Infallible>(Event::default().id(cursor.to_string()).event("team_progress").data(event.to_string()));
}
tokio::select! {biased;_=app.stop.cancelled()=>break,_=tokio::time::sleep(Duration::from_millis(250))=>{}}
match app.store.team_events(&id,cursor) {
Ok(next)=>batch=next,
Err(_)=>{yield Ok(Event::default().event("stream_error").data("{\"error\":\"reconnect_and_reconcile\"}"));break;}
}
}
};
Ok(Sse::new(stream)
.keep_alive(KeepAlive::new().interval(Duration::from_secs(10)))
.into_response())
}
pub(crate) async fn events(
Extension(app): Extension<App>,
Path(sid): Path<String>,
Query(query): Query<Cursor>,
headers: HeaderMap,
) -> Result<Response, ApiError> {
let header = headers
.get("last-event-id")
.map(|v| {
v.to_str()
.ok()
.and_then(|v| v.parse::<i64>().ok())
.ok_or_else(|| ApiError(SubmissionError::InvalidRequest.into()))
})
.transpose()?;
let mut cursor = header.or(query.after).unwrap_or(0);
let initial = app.store.read_events(&sid, cursor)?;
let permit = app
.streams
.clone()
.try_acquire_owned()
.map_err(|_| ApiError(SubmissionError::QueueFull.into()))?;
let stream = async_stream::stream! {
let _permit=permit;
let mut batch=initial;
loop {
for event in batch.drain(..) {
cursor=event.event_id;
yield Ok::<_,Infallible>(Event::default().id(cursor.to_string()).event("progress").data(serde_json::to_string(&event).expect("JSON event")));
}
tokio::select! {biased; _=app.stop.cancelled()=>break, _=tokio::time::sleep(Duration::from_millis(250))=>{} }
match app.store.read_events(&sid,cursor) {
Ok(next)=>batch=next,
Err(_)=>{yield Ok(Event::default().event("stream_error").data("{\"error\":\"reconnect_and_reconcile\"}"));break;}
}
}
};
Ok(Sse::new(stream)
.keep_alive(KeepAlive::new().interval(Duration::from_secs(10)))
.into_response())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn permanent_control_errors_are_not_service_outages() {
assert_eq!(
ApiError(crate::git_access::Unavailable.into())
.into_response()
.status(),
StatusCode::BAD_REQUEST
);
for (error, status) in [
(SubmissionError::InvalidRequest, StatusCode::BAD_REQUEST),
(SubmissionError::NotFound, StatusCode::NOT_FOUND),
(SubmissionError::Conflict, StatusCode::CONFLICT),
(SubmissionError::ControlConflict, StatusCode::CONFLICT),
(SubmissionError::Closed, StatusCode::CONFLICT),
(SubmissionError::ProviderBlocked, StatusCode::CONFLICT),
(SubmissionError::QueueFull, StatusCode::TOO_MANY_REQUESTS),
] {
assert_eq!(ApiError(error.into()).into_response().status(), status);
}
assert_eq!(
ApiError(anyhow::anyhow!("unknown database outcome"))
.into_response()
.status(),
StatusCode::SERVICE_UNAVAILABLE
);
}
}