Skip to content
Python
20.4 KB
Raw Blame
//! Durable server teams. All model identities come from their fenced session.
use crate::{
    auth,
    db::{self, Database},
    PostgresStore,
};
use anyhow::{ensure, Context, Result};
use openworker_runtime::{
    store::Event,
    teams::{Board, Item, Member, Role},
};
use postgres::Transaction;
use serde_json::{json, Value};

pub use openworker_runtime::team_service::{Command, Create, Error};
fn request_error(error: anyhow::Error) -> anyhow::Error {
    if error.downcast_ref::<postgres::Error>().is_some()
        || error.downcast_ref::<auth::Error>().is_some()
        || error.downcast_ref::<crate::SubmissionError>().is_some()
        || error.downcast_ref::<Error>().is_some()
    {
        error
    } else {
        Error::Invalid.into()
    }
}

pub(crate) fn migrate(tx: &mut Transaction<'_>) -> Result<()> {
    if tx
        .query_one("SELECT to_regclass('team_version') IS NOT NULL", &[])?
        .get::<_, bool>(0)
    {
        let rows = tx.query("SELECT version FROM team_version WHERE singleton", &[])?;
        ensure!(
            rows.len() == 1 && rows[0].get::<_, i32>(0) == 1,
            "Unsupported team schema"
        );
    } else {
        ensure!(
            !tx.query_one("SELECT to_regclass('teams') IS NOT NULL", &[])?
                .get::<_, bool>(0),
            "Unversioned team store"
        );
        tx.batch_execute("CREATE TABLE team_version(singleton BOOLEAN PRIMARY KEY CHECK(singleton),version INTEGER NOT NULL);
          INSERT INTO team_version VALUES(TRUE,1);
          CREATE TABLE teams(id TEXT PRIMARY KEY, request_id TEXT NOT NULL, actor TEXT NOT NULL, parameters JSONB NOT NULL, data JSONB NOT NULL, cursor BIGINT NOT NULL DEFAULT 0, UNIQUE(actor,request_id));
          CREATE TABLE team_members(session_id TEXT PRIMARY KEY REFERENCES sessions(id),team_id TEXT NOT NULL REFERENCES teams(id));
          CREATE TABLE team_events(team_id TEXT NOT NULL REFERENCES teams(id),cursor BIGINT NOT NULL,kind TEXT NOT NULL,data JSONB NOT NULL,PRIMARY KEY(team_id,cursor));
          CREATE TABLE team_commands(team_id TEXT NOT NULL REFERENCES teams(id),id TEXT NOT NULL,actor TEXT NOT NULL,request JSONB NOT NULL,result JSONB NOT NULL,PRIMARY KEY(team_id,id));
          CREATE TABLE team_deliveries(session_id TEXT NOT NULL,submission_id TEXT NOT NULL,team_id TEXT NOT NULL REFERENCES teams(id),item_id TEXT,assignment BIGINT,PRIMARY KEY(session_id,submission_id),FOREIGN KEY(session_id,submission_id) REFERENCES submissions(session_id,id));
          CREATE TABLE team_completions(session_id TEXT NOT NULL,submission_id TEXT NOT NULL,team_id TEXT NOT NULL REFERENCES teams(id),status TEXT NOT NULL,processed BOOLEAN NOT NULL DEFAULT FALSE,PRIMARY KEY(session_id,submission_id));")?;
    }
    Ok(())
}
pub(crate) fn present(tx: &mut Transaction<'_>) -> Result<bool> {
    Ok(tx
        .query_one("SELECT to_regclass('team_version') IS NOT NULL", &[])?
        .get(0))
}
pub(crate) fn load(tx: &mut Transaction<'_>, id: &str, lock: bool) -> Result<Board> {
    let sql = if lock {
        "SELECT data FROM teams WHERE id=$1 FOR UPDATE"
    } else {
        "SELECT data FROM teams WHERE id=$1"
    };
    let row = tx.query_opt(sql, &[&id])?.ok_or(Error::NotFound)?;
    Ok(serde_json::from_value(row.get(0))?)
}
pub(crate) fn event(
    tx: &mut Transaction<'_>,
    board: &Board,
    kind: &str,
    data: Value,
) -> Result<i64> {
    let value = serde_json::to_value(board)?;
    ensure!(
        // Keep room for a full board read nested inside a canonical tool result.
        value.to_string().len() <= 200000,
        "Team board storage limit reached"
    );
    let cursor:i64=tx.query_opt("UPDATE teams SET data=$2,cursor=cursor+1 WHERE id=$1 AND cursor<10000 RETURNING cursor",&[&board.id,&value])?.ok_or(Error::Limit)?.get(0);
    tx.execute(
        "INSERT INTO team_events VALUES($1,$2,$3,$4)",
        &[&board.id, &cursor, &kind, &data],
    )?;
    Ok(cursor)
}
fn read_events(
    tx: &mut Transaction<'_>,
    board: &Board,
    after: i64,
    limit: i64,
    item: Option<&str>,
    messages: bool,
) -> Result<Value> {
    let cursor: i64 = tx
        .query_one("SELECT cursor FROM teams WHERE id=$1", &[&board.id])?
        .get(0);
    ensure!(
        after >= 0 && after <= cursor && (1..=50).contains(&limit),
        "Invalid team cursor or page size"
    );
    let rows=tx.query("SELECT cursor,kind,data FROM team_events WHERE team_id=$1 AND cursor>$2 ORDER BY cursor LIMIT $3",&[&board.id,&after,&limit])?;
    let mut next = after;
    let mut events = vec![];
    for row in rows {
        let seq: i64 = row.get(0);
        let kind: String = row.get(1);
        let data: Value = row.get(2);
        ensure!(seq == next + 1, "Team event history incomplete");
        next = seq;
        if messages && kind != "message_posted" {
            continue;
        }
        if item.is_some_and(|i| !i.is_empty() && data["item_id"] != i) {
            continue;
        }
        events.push(json!({"cursor":seq,"type":kind,"data":data}));
    }
    ensure!(
        next > after || after == cursor,
        "Team event history incomplete"
    );
    Ok(json!({"events":events,"next_after":next,"has_more":next<cursor,"cursor":cursor}))
}

impl Database {
    pub(crate) fn team_create(&mut self, request: Create) -> Result<Value> {
        crate::submissions::validate_id(&request.request_id, 100)?;
        let board = openworker_runtime::team_service::new_board(&request)?;
        let lead = board.lead.clone();
        let members = &board.members;
        let mut tx = self.client.transaction()?;
        let identity = auth::guard(
            &mut tx,
            self.auth.as_ref(),
            auth::Action::Control,
            Some(&lead),
            None,
        )?;
        let actor = identity
            .as_ref()
            .map_or("trusted-host", |p| p.principal.as_str());
        // Serialize duplicate creations before allocating membership.
        tx.query_one(
            "SELECT pg_advisory_xact_lock(hashtext($1))",
            &[&format!("team-create:{actor}:{}", request.request_id)],
        )?;
        let parameters = serde_json::to_value(&request)?;
        if let Some(row) = tx.query_opt(
            "SELECT id,parameters FROM teams WHERE actor=$1 AND request_id=$2",
            &[&actor, &request.request_id],
        )? {
            ensure!(row.get::<_, Value>(1) == parameters, Error::Conflict);
            let id: String = row.get(0);
            tx.commit()?;
            return Ok(json!({"team_id":id,"duplicate":true}));
        }
        let mut principals = std::collections::BTreeSet::new();
        for m in members.values() {
            auth::guard(
                &mut tx,
                self.auth.as_ref(),
                auth::Action::Control,
                Some(&m.session_id),
                None,
            )?;
            let s = db::locked(&mut tx, &m.session_id, None)?;
            openworker_runtime::team_service::fresh_member(&s, m.role)?;
            ensure!(
                !tx.query_one(
                    "SELECT EXISTS(SELECT 1 FROM team_members WHERE session_id=$1)",
                    &[&m.session_id]
                )?
                .get::<_, bool>(0),
                "Session already belongs to a team"
            );
            for row in tx.query(
                "SELECT principal FROM api_session_owners WHERE session_id=$1",
                &[&m.session_id],
            )? {
                principals.insert(row.get::<_, String>(0));
            }
        }
        ensure!(
            principals.len() <= 1,
            "Team members must share an application owner"
        );
        tx.execute(
            "INSERT INTO teams(id,request_id,actor,parameters,data) VALUES($1,$2,$3,$4,$5)",
            &[
                &board.id,
                &request.request_id,
                &actor,
                &parameters,
                &json!(board),
            ],
        )?;
        for sid in board.members.keys() {
            tx.execute("INSERT INTO team_members VALUES($1,$2)", &[&sid, &board.id])?;
        }
        let cursor = event(
            &mut tx,
            &board,
            "team_created",
            json!({"actor":actor,"members":board.members}),
        )?;
        auth::audit(
            &mut tx,
            identity.as_ref(),
            auth::Action::Control,
            Some(&board.lead),
            Some(&request.request_id),
        )?;
        tx.commit()?;
        Ok(json!({"team_id":board.id,"cursor":cursor,"duplicate":false}))
    }
    pub(crate) fn team_snapshot(&mut self, id: &str) -> Result<Value> {
        let mut tx = self.client.transaction()?;
        let board = load(&mut tx, id, false)?;
        auth::guard(
            &mut tx,
            self.auth.as_ref(),
            auth::Action::Read,
            Some(&board.lead),
            None,
        )?;
        // Row lock keeps the snapshot and cursor coherent with board writes.
        let board = load(&mut tx, id, true)?;
        let cursor: i64 = tx
            .query_one("SELECT cursor FROM teams WHERE id=$1", &[&id])?
            .get(0);
        let pending: bool = tx.query_one(
            "SELECT EXISTS(SELECT 1 FROM team_deliveries d JOIN submissions s ON s.session_id=d.session_id AND s.id=d.submission_id WHERE d.team_id=$1 AND s.status NOT IN ('completed','failed','cancelled','interrupted','truncated','blocked')) OR EXISTS(SELECT 1 FROM team_completions WHERE team_id=$1 AND NOT processed)",
            &[&id],
        )?.get(0);
        tx.commit()?;
        Ok(json!({"board":board,"cursor":cursor,"pending_work":pending}))
    }
    pub(crate) fn team_events(&mut self, id: &str, after: i64) -> Result<Value> {
        let mut tx = self.client.transaction()?;
        let board = load(&mut tx, id, true)?;
        auth::guard(
            &mut tx,
            self.auth.as_ref(),
            auth::Action::Read,
            Some(&board.lead),
            None,
        )?;
        let result = read_events(&mut tx, &board, after, 50, None, false)?;
        tx.commit()?;
        Ok(result)
    }
    pub(crate) fn team_command(&mut self, id: &str, command: Command) -> Result<Value> {
        crate::submissions::validate_id(&command.request_id, 100)?;
        let mut tx = self.client.transaction()?;
        let mut board = load(&mut tx, id, true)?;
        let identity = auth::guard(
            &mut tx,
            self.auth.as_ref(),
            auth::Action::Control,
            Some(&board.lead),
            None,
        )?;
        for sid in board.members.keys() {
            auth::guard(
                &mut tx,
                self.auth.as_ref(),
                auth::Action::Control,
                Some(sid),
                None,
            )?;
        }
        let actor = format!(
            "app:{}",
            identity
                .as_ref()
                .map_or("trusted-host", |p| p.principal.as_str())
        );
        let key = format!("app:{}", command.request_id);
        let raw = json!(command);
        if let Some(row) = tx.query_opt(
            "SELECT actor,request,result FROM team_commands WHERE team_id=$1 AND id=$2",
            &[&id, &key],
        )? {
            ensure!(
                row.get::<_, String>(0) == actor && row.get::<_, Value>(1) == raw,
                Error::Conflict
            );
            let result: Value = row.get(2);
            tx.commit()?;
            return Ok(result);
        }
        // Reads have dedicated endpoints. Bound idempotency records as well as
        // board events so app-issued read commands cannot grow storage forever.
        ensure!(
            tx.query_one(
                "SELECT count(*) FROM team_commands WHERE team_id=$1",
                &[&id]
            )?
            .get::<_, i64>(0)
                < 20000,
            Error::Limit
        );
        let result = if command.operation == "team_add_member" {
            ensure!(!board.cancelled, "Team cancelled");
            let member: Member = serde_json::from_value(command.arguments.clone())?;
            openworker_runtime::team_service::additional_member(&board, &member)?;
            auth::guard(
                &mut tx,
                self.auth.as_ref(),
                auth::Action::Control,
                Some(&member.session_id),
                None,
            )?;
            let session = db::locked(&mut tx, &member.session_id, None)?;
            openworker_runtime::team_service::fresh_member(&session, member.role)?;
            ensure!(
                !tx.query_one(
                    "SELECT EXISTS(SELECT 1 FROM team_members WHERE session_id=$1)",
                    &[&member.session_id]
                )?
                .get::<_, bool>(0),
                "Session already belongs to a team"
            );
            let owners = tx.query(
                "SELECT DISTINCT principal FROM api_session_owners WHERE session_id IN ($1,$2)",
                &[&board.lead, &member.session_id],
            )?;
            ensure!(
                owners.len() <= 1,
                "Team members must share an application owner"
            );
            tx.execute(
                "INSERT INTO team_members VALUES($1,$2)",
                &[&member.session_id, &id],
            )?;
            board
                .members
                .insert(member.session_id.clone(), member.clone());
            let cursor = event(
                &mut tx,
                &board,
                "member_added",
                json!({"actor":actor,"member":member}),
            )?;
            json!({"cursor":cursor,"member":member})
        } else if command.operation == "team_cancel" {
            ensure!(
                command.arguments == json!({}),
                "Cancel accepts no arguments"
            );
            crate::team_execution::cancel(&mut tx, &mut board, &actor)?;
            let cursor = event(&mut tx, &board, "team_cancelled", json!({"actor":actor}))?;
            json!({"cursor":cursor,"cancelled":true})
        } else {
            openworker_runtime::team_service::operate(
                &mut PgTeamTx(&mut tx),
                &mut board,
                &actor,
                Role::Lead,
                &command.operation,
                &command.arguments,
                self.max_queued_turns,
            )?
        };
        tx.execute(
            "INSERT INTO team_commands VALUES($1,$2,$3,$4,$5)",
            &[&id, &key, &actor, &raw, &result],
        )?;
        auth::audit(
            &mut tx,
            identity.as_ref(),
            auth::Action::Control,
            Some(&board.lead),
            Some(&command.request_id),
        )?;
        tx.commit()?;
        Ok(result)
    }
    pub(crate) fn team_context(
        &mut self,
        sid: &str,
    ) -> Result<Option<openworker_runtime::teams::Context>> {
        let mut tx = self.client.transaction()?;
        if !present(&mut tx)? {
            return Ok(None);
        }
        let result = if let Some(row) = tx.query_opt(
            "SELECT team_id FROM team_members WHERE session_id=$1",
            &[&sid],
        )? {
            let id: String = row.get(0);
            let board = load(&mut tx, &id, false)?;
            let m = board
                .members
                .get(sid)
                .context("Team membership inconsistent")?;
            let delivery=tx.query_opt("SELECT d.item_id,d.assignment FROM team_deliveries d JOIN sessions s ON s.id=d.session_id WHERE d.session_id=$1 AND d.submission_id=s.data->'turn'->>'submission_id'",&[&sid])?;
            Some(openworker_runtime::teams::Context {
                team_id: id,
                role: m.role,
                session_id: sid.into(),
                active_item: delivery.as_ref().and_then(|r| r.get(0)),
                assignment: delivery.as_ref().and_then(|r| r.get(1)),
            })
        } else {
            None
        };
        tx.commit()?;
        Ok(result)
    }
    pub(crate) fn team_call(
        &mut self,
        sid: &str,
        tid: &str,
        call: Value,
        args: Value,
    ) -> Result<Event> {
        let epoch = self.epoch(sid)?;
        let mut tx = self.client.transaction()?;
        let team: String = tx
            .query_opt(
                "SELECT team_id FROM team_members WHERE session_id=$1",
                &[&sid],
            )?
            .context("Session is not in a team")?
            .get(0);
        let mut board = load(&mut tx, &team, true)?;
        let mut s = db::locked(&mut tx, sid, Some((&self.owner, epoch)))?;
        db::read_history(&mut tx, &mut s)?;
        let t = s.turn.as_ref().context("No turn")?;
        ensure!(
            t.id == tid && t.status == "running" && t.queue.first() == Some(&call),
            "Team call stale or cancelled"
        );
        let role = board
            .members
            .get(sid)
            .context("Team membership unavailable")?
            .role;
        ensure!(s.package.team_role == Some(role), "Coworker role changed");
        let name = call["name"].as_str().context("Tool name missing")?;
        crate::controls::before_event(&mut tx, &s, "tool_started", &json!({}))?;
        let key = format!(
            "agent:{sid}:{tid}:{}",
            call["call_id"].as_str().context("Call ID missing")?
        );
        let output = {
            let mut operation = tx.transaction()?;
            match openworker_runtime::team_service::operate(
                &mut PgTeamTx(&mut operation),
                &mut board,
                sid,
                role,
                name,
                &args,
                self.max_queued_turns,
            ) {
                Ok(v) => {
                    operation.commit()?;
                    v
                }
                Err(e) => {
                    operation.rollback()?;
                    if e.downcast_ref::<postgres::Error>().is_some() {
                        return Err(e);
                    }
                    json!({"isError":true,"message":e.to_string()})
                }
            }
        };
        tx.execute(
            "INSERT INTO team_commands VALUES($1,$2,$3,$4,$5)",
            &[&team, &key, &sid, &call, &output],
        )?;
        if !openworker_runtime::safety::hold_internal_result(&mut s, &call) {
            s.history.push(json!({"type":"function_call_output","call_id":call["call_id"],"output":output.to_string()}));
            s.turn.as_mut().unwrap().queue.remove(0);
        }
        db::append_history(&mut tx, &s)?;
        db::write_metadata(&mut tx, &s)?;
        let mut event = db::event(
            &mut tx,
            &s,
            &self.owner,
            epoch,
            "tool_finished",
            openworker_runtime::safety::public_internal_event(
                &s,
                json!({"name":name,"call_id":call["call_id"],"result":output,"execution_kind":"internal","team_id":team,"enforcement":"team-role-and-session-scope"}),
            ),
        )?;
        event.data["result"] = output;
        tx.commit()?;
        Ok(event)
    }
}
impl PostgresStore {
    pub fn create_team(&self, request: Create) -> Result<Value> {
        self.call(move |db| db.team_create(request).map_err(request_error))
    }
    pub fn team_snapshot(&self, id: &str) -> Result<Value> {
        let id = id.to_owned();
        self.call(move |db| db.team_snapshot(&id).map_err(request_error))
    }
    pub fn team_events(&self, id: &str, after: i64) -> Result<Value> {
        let id = id.to_owned();
        self.call(move |db| db.team_events(&id, after).map_err(request_error))
    }
    pub fn team_command(&self, id: &str, command: Command) -> Result<Value> {
        let id = id.to_owned();
        self.call(move |db| db.team_command(&id, command).map_err(request_error))
    }
}

struct PgTeamTx<'a, 'b>(&'a mut Transaction<'b>);
impl openworker_runtime::team_service::Transaction for PgTeamTx<'_, '_> {
    fn read_events(
        &mut self,
        b: &Board,
        a: i64,
        l: i64,
        i: Option<&str>,
        m: bool,
    ) -> Result<Value> {
        read_events(self.0, b, a, l, i, m)
    }
    fn assignment_finished(&mut self, i: &Item) -> Result<()> {
        crate::team_execution::assignment_finished(self.0, i)
    }
    fn current_assignment(&mut self, s: &str, i: &Item) -> Result<()> {
        crate::team_execution::current_assignment(self.0, s, i)
    }
    fn deliver(
        &mut self,
        b: &mut Board,
        s: &str,
        i: Option<&Item>,
        t: &str,
        l: u16,
    ) -> Result<String> {
        crate::team_execution::deliver(self.0, b, s, i, t, l)
    }
    fn event(&mut self, b: &Board, k: &str, d: Value) -> Result<i64> {
        event(self.0, b, k, d)
    }
}