Skip to content
Python
13.9 KB
Raw Blame
//! Authentication belongs to execution storage, never the recording database.
use crate::http::{ApiError, App};
use axum::{
    extract::{Path, Query},
    routing::get,
    Extension, Json, Router,
};
use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine};
use openworker_observability::{
    postgres::Postgres,
    query::{Query as Read, Scope},
    summary::{self, Canonical},
    Store,
};
use openworker_store_postgres::{
    auth::Action,
    run_query::{self, Run},
};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};

#[derive(Debug)]
pub(crate) enum Error {
    Invalid,
    Unavailable,
    Busy,
}
impl std::fmt::Display for Error {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "Recording query failed")
    }
}
impl std::error::Error for Error {}
pub(crate) fn authority(method: &str, path: &str) -> bool {
    if method != "GET" {
        return false;
    }
    let path = path
        .strip_prefix("/v1/")
        .map(|p| format!("/{p}"))
        .unwrap_or_else(|| path.into());
    matches!(
        path.split('/').collect::<Vec<_>>().as_slice(),
        ["", "observability", "runs" | "summary" | "health"]
            | [
                "",
                "sessions",
                _,
                "observability",
                _,
                "timeline" | "summary" | "children" | "subtree-summary"
            ]
    )
}
pub(crate) fn action(method: &str, path: &str) -> Option<Action> {
    authority(method, path).then_some({
        if matches!(path, "/observability/health" | "/v1/observability/health") {
            Action::Admin
        } else {
            Action::Read
        }
    })
}
pub(crate) fn router(prefix: &str) -> Router<App> {
    Router::new()
        .route(&format!("{prefix}/observability/health"), get(health))
        .route(&format!("{prefix}/observability/runs"), get(search))
        .route(&format!("{prefix}/observability/summary"), get(grouped))
        .route(
            &format!("{prefix}/sessions/:sid/observability/:turn/timeline"),
            get(timeline),
        )
        .route(
            &format!("{prefix}/sessions/:sid/observability/:turn/children"),
            get(children),
        )
        .route(
            &format!("{prefix}/sessions/:sid/observability/:turn/subtree-summary"),
            get(subtree_summary),
        )
        .route(
            &format!("{prefix}/sessions/:sid/observability/:turn/summary"),
            get(summary),
        )
}
async fn health(Extension(app): Extension<App>) -> Result<Json<Value>, ApiError> {
    // These counters cover the serving API process, not an authorized run or
    // the cluster. Never expose other principals' traffic through a read token.
    app.store.authorize(Action::Admin, None, None)?;
    Ok(Json(json!({
        "scope":"api_instance", "recording_enabled":app.config.observability.is_some(),
        "recording":app.config.recording_stats.as_ref().map(|s|s.snapshot()),
        "exporter_enabled":app.runtime.telemetry.enabled(),
        "exporter":app.runtime.telemetry.snapshot()
    })))
}
#[derive(Clone, Default, Deserialize)]
#[serde(deny_unknown_fields)]
struct Search {
    q: Option<String>,
    agent: Option<String>,
    from_ms: Option<i64>,
    to_ms: Option<i64>,
    limit: Option<u16>,
    after: Option<String>,
}
#[derive(Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct Cursor {
    version: u8,
    binding: String,
    as_of_us: i64,
    after: run_query::Cursor,
}
fn run_search(app: &App, q: &Search) -> Result<Value, ApiError> {
    let identity = app.store.authorize(Action::Read, None, None)?;
    let binding = openworker_runtime::bundle::digest(
        json!([
            app.config.schema,
            app.service.as_ref().map(|s| (&s.id, &s.namespace)),
            identity.credential_id,
            q.q,
            q.agent,
            q.from_ms,
            q.to_ms
        ])
        .to_string()
        .as_bytes(),
    );
    let cursor = q
        .after
        .as_ref()
        .map(|s| {
            if s.len() > 1024 {
                return Err(Error::Invalid);
            }
            let bytes = URL_SAFE_NO_PAD.decode(s).map_err(|_| Error::Invalid)?;
            let cursor: Cursor = serde_json::from_slice(&bytes).map_err(|_| Error::Invalid)?;
            if cursor.version != 1 || cursor.binding != binding {
                return Err(Error::Invalid);
            }
            Ok(cursor)
        })
        .transpose()
        .map_err(|e| ApiError(e.into()))?;
    let page = app.store.search_runs(run_query::Search {
        text: q.q.clone().unwrap_or_default(),
        agent: q.agent.clone(),
        from_ms: q.from_ms,
        to_ms: q.to_ms,
        after: cursor.as_ref().map(|c| c.after.clone()),
        as_of_us: cursor.as_ref().map(|c| c.as_of_us),
        limit: q.limit.unwrap_or(25),
    })?;
    let next = page.next.map(|after| {
        URL_SAFE_NO_PAD.encode(
            serde_json::to_vec(&Cursor {
                version: 1,
                binding,
                as_of_us: page.as_of_us,
                after,
            })
            .expect("typed cursor"),
        )
    });
    Ok(json!({"runs":page.runs,"next_cursor":next,"as_of_us":page.as_of_us}))
}
async fn search(
    Extension(app): Extension<App>,
    Query(q): Query<Search>,
) -> Result<Json<Value>, ApiError> {
    Ok(Json(run_search(&app, &q)?))
}
fn scope(sid: &str, turn: &str) -> Scope {
    Scope {
        session_hash: openworker_runtime::bundle::digest(sid.as_bytes()),
        turn_hash: openworker_runtime::bundle::digest(turn.as_bytes()),
    }
}
fn canonical(run: &Run) -> Canonical {
    Canonical {
        terminal: openworker_runtime::store::terminal(&run.status),
        expected_segments: run.observation.as_ref().map(|o| o.segments),
        first_started_ms: run.observation.as_ref().map(|o| o.first_started_at_ms),
        last_span_id: run.observation.as_ref().map(|o| {
            o.last_segment
                .span
                .iter()
                .map(|b| format!("{b:02x}"))
                .collect()
        }),
        pending_waits: Some(run.pending_waits),
    }
}
async fn read(app: &App, q: Read) -> Result<openworker_observability::query::Page, ApiError> {
    q.validate().map_err(|_| ApiError(Error::Invalid.into()))?;
    let options = app
        .config
        .observability
        .as_ref()
        .ok_or_else(|| ApiError(Error::Unavailable.into()))?;
    let config = options
        .database(&app.config)
        .map_err(|_| ApiError(Error::Unavailable.into()))?;
    let permit = app
        .recording_readers
        .clone()
        .try_acquire_owned()
        .map_err(|_| ApiError(Error::Busy.into()))?;
    // The permit stays with the blocking job even if the HTTP client disconnects.
    tokio::task::spawn_blocking(move || {
        let _permit = permit;
        Postgres::open(&config)?.timeline(&q)
    })
    .await
    .map_err(|_| ApiError(Error::Unavailable.into()))?
    .map_err(|_| ApiError(Error::Unavailable.into()))
}
#[derive(Default, Deserialize)]
#[serde(deny_unknown_fields)]
struct Page {
    after: Option<u64>,
    through: Option<u64>,
    limit: Option<u16>,
}
async fn timeline(
    Extension(app): Extension<App>,
    Path((sid, turn)): Path<(String, String)>,
    Query(q): Query<Page>,
) -> Result<Json<Value>, ApiError> {
    app.store.observation_run(&sid, &turn)?; // BEFORE configuration/recording access.
    let limit = q.limit.unwrap_or(100);
    if !(1..=200).contains(&limit) {
        return Err(ApiError(Error::Invalid.into()));
    }
    let page = read(
        &app,
        Read {
            deployment: app.config.schema.clone(),
            scopes: vec![scope(&sid, &turn)],
            after: q.after.unwrap_or(0),
            through: q.through,
            limit,
        },
    )
    .await?;
    let run = app.store.observation_run(&sid, &turn)?; // Revocation/scope recheck after read.
    Ok(Json(
        json!({"run":run,"page":page,"coverage":{"guarantee":"best_effort","order":"recording_cursor","snapshot_across_requests":false}}),
    ))
}
#[derive(Default, Deserialize)]
#[serde(deny_unknown_fields)]
struct Summary {
    through: Option<u64>,
}
async fn summary(
    Extension(app): Extension<App>,
    Path((sid, turn)): Path<(String, String)>,
    Query(q): Query<Summary>,
) -> Result<Json<Value>, ApiError> {
    app.store.observation_run(&sid, &turn)?;
    let page = read(
        &app,
        Read {
            deployment: app.config.schema.clone(),
            scopes: vec![scope(&sid, &turn)],
            after: 0,
            through: q.through,
            limit: openworker_observability::query::MAX_SUMMARY_STEPS,
        },
    )
    .await?;
    let run = app.store.observation_run(&sid, &turn)?;
    let summary = summary::build(&page, canonical(&run));
    Ok(Json(json!({"run":run,"summary":summary})))
}
async fn subtree_summary(
    Extension(app): Extension<App>,
    Path((sid, turn)): Path<(String, String)>,
    Query(q): Query<Summary>,
) -> Result<Json<Value>, ApiError> {
    use openworker_runtime::observation_links::{subtree, Run as LinkRun};
    app.store.observation_run(&sid, &turn)?;
    let selection = subtree(
        LinkRun {
            session_id: sid,
            turn_id: turn,
        },
        |r| app.store.observation_children(&r.session_id, &r.turn_id),
    )?;
    for run in &selection.runs {
        app.store.observation_run(&run.session_id, &run.turn_id)?;
    }
    let page = read(
        &app,
        Read {
            deployment: app.config.schema.clone(),
            scopes: selection
                .runs
                .iter()
                .map(|r| scope(&r.session_id, &r.turn_id))
                .collect(),
            after: 0,
            through: q.through,
            limit: openworker_observability::query::MAX_SUMMARY_STEPS,
        },
    )
    .await?;
    for run in &selection.runs {
        app.store.observation_run(&run.session_id, &run.turn_id)?;
    }
    Ok(Json(
        json!({"selection":selection,"summary":summary::build(&page,Canonical::default())}),
    ))
}
async fn grouped(
    Extension(app): Extension<App>,
    Query(q): Query<Search>,
) -> Result<Json<Value>, ApiError> {
    if q.limit.unwrap_or(25) > 25 {
        return Err(ApiError(Error::Invalid.into()));
    }
    let selected = run_search(&app, &q)?;
    let runs: Vec<Run> =
        serde_json::from_value(selected["runs"].clone()).map_err(|e| ApiError(e.into()))?;
    if runs.is_empty() {
        let summary = summary::build(
            &openworker_observability::query::Page {
                steps: vec![],
                through: 0,
                next: None,
            },
            Canonical::default(),
        );
        return Ok(Json(
            json!({"selection":selected,"summary":summary,"scope":"selected_run_page","selection_truncated":false,"selected_runs":0,"runs_with_records":0}),
        ));
    }
    let scopes = runs
        .iter()
        .map(|r| scope(&r.session_id, &r.turn_id))
        .collect();
    let page = read(
        &app,
        Read {
            deployment: app.config.schema.clone(),
            scopes,
            after: 0,
            through: None,
            limit: openworker_observability::query::MAX_SUMMARY_STEPS,
        },
    )
    .await?;
    let mut pending = Some(0u64);
    for run in &runs {
        let latest = app.store.observation_run(&run.session_id, &run.turn_id)?;
        pending = pending.and_then(|n| n.checked_add(latest.pending_waits));
    }
    // Multiple concurrent runs have no single meaningful turn wall elapsed.
    let recorded: std::collections::BTreeSet<_> = page
        .steps
        .iter()
        .map(|r| (&r.step.session_hash, &r.step.turn_hash))
        .collect();
    let recorded_runs = recorded.len();
    let summary = summary::build(
        &page,
        Canonical {
            pending_waits: pending,
            ..Default::default()
        },
    );
    Ok(Json(
        json!({"selection":selected,"summary":summary,"scope":"selected_run_page","selection_truncated":selected["next_cursor"].is_string(),"selected_runs":runs.len(),"runs_with_records":recorded_runs}),
    ))
}

async fn children(
    Extension(app): Extension<App>,
    Path((sid, turn)): Path<(String, String)>,
) -> Result<Json<Value>, ApiError> {
    app.store.observation_run(&sid, &turn)?;
    let children = app.store.observation_children(&sid, &turn)?;
    app.store.observation_run(&sid, &turn)?;
    Ok(Json(
        json!({"children":children,"coverage":"authorized_captured_spawn_origins_only"}),
    ))
}

#[cfg(test)]
mod tests {
    use super::*;
    #[test]
    fn only_exact_read_routes_are_authorized() {
        for prefix in ["", "/v1"] {
            for path in [
                "/observability/runs",
                "/observability/summary",
                "/observability/health",
                "/sessions/s/observability/t/timeline",
                "/sessions/s/observability/t/summary",
                "/sessions/s/observability/t/children",
                "/sessions/s/observability/t/subtree-summary",
            ] {
                let path = format!("{prefix}{path}");
                assert!(authority("GET", &path));
                for method in ["POST", "PUT", "DELETE"] {
                    assert!(!authority(method, &path));
                }
            }
        }
        assert!(matches!(
            action("GET", "/v1/observability/health"),
            Some(Action::Admin)
        ));
        for path in [
            "/v1evil/observability/runs",
            "/v1/observability/runs/extra",
            "/v1/sessions/s/observability/t/anything",
            "/v1/sessions/s/observability/t/summary/extra",
        ] {
            assert!(!authority("GET", path));
        }
    }
    #[test]
    fn callers_cannot_choose_recording_database_or_scope_hashes() {
        assert!(serde_json::from_value::<Search>(json!({"deployment":"other"})).is_err());
        assert!(serde_json::from_value::<Page>(json!({"session_hash":"other"})).is_err());
        assert!(serde_json::from_value::<Summary>(json!({"database":"other"})).is_err());
    }
}