Skip to content
Python
10.1 KB
Raw Blame
//! Team deliveries use the ordinary turn queue. Never replay uncertain effects.
use anyhow::{ensure, Context, Result};
use openworker_runtime::{
    store::{terminal, Session},
    teams::{Board, Item, Status},
};
use postgres::Transaction;
use serde_json::{json, Value};

pub(crate) fn finished(tx: &mut Transaction<'_>, s: &Session) -> Result<()> {
    if !crate::teams::present(tx)? {
        return Ok(());
    }
    let Some(t) = &s.turn else {
        return Ok(());
    };
    tx.execute("INSERT INTO team_completions(session_id,submission_id,team_id,status) SELECT session_id,submission_id,team_id,$3 FROM team_deliveries WHERE session_id=$1 AND submission_id=$2 AND item_id IS NOT NULL ON CONFLICT DO NOTHING",&[&s.id,&t.submission_id,&t.status])?;
    Ok(())
}
impl crate::db::Database {
    /// Durable completion outbox: completion commit never locks another session.
    /// An idle scheduler slot can notify the lead after crash/restart exactly once
    /// at the submission level; delivery does not imply exactly-once model billing.
    pub(crate) fn dispatch_team_completion(&mut self) -> Result<()> {
        let mut tx = self.client.transaction()?;
        if !crate::teams::present(&mut tx)? {
            return Ok(());
        }
        let Some(row)=tx.query_opt("SELECT team_id FROM team_completions WHERE NOT processed ORDER BY team_id,session_id,submission_id LIMIT 1",&[])? else{return Ok(());};
        let team: String = row.get(0);
        let mut board = crate::teams::load(&mut tx, &team, true)?;
        let Some(row)=tx.query_opt("SELECT session_id,submission_id,status FROM team_completions WHERE team_id=$1 AND NOT processed ORDER BY session_id,submission_id LIMIT 1 FOR UPDATE SKIP LOCKED",&[&team])? else{return Ok(());};
        let (sid, key, status): (String, String, String) = (row.get(0), row.get(1), row.get(2));
        if let Some(item) =
            openworker_runtime::team_service::complete(&mut board, &sid, &key, &status)
        {
            let id = item.id.clone();
            let item_snapshot = json!(item);
            let lead = board.lead.clone();
            let lead_session = crate::db::locked(&mut tx, &lead, None)?;
            let queued: i64 = tx
                .query_one(
                    "SELECT count(*) FROM submissions WHERE session_id=$1 AND status='queued'",
                    &[&lead],
                )?
                .get(0);
            let notified = board.deliveries < 128
                && queued < i64::from(self.max_queued_turns)
                && !lead_session.closed
                && !lead_session.provider_blocked;
            if notified {
                deliver(
                    &mut tx,
                    &mut board,
                    &lead,
                    None,
                    &openworker_runtime::team_service::completion_text(&sid, &id, &status),
                    self.max_queued_turns,
                )?;
            }
            crate::teams::event(
                &mut tx,
                &board,
                "assignment_turn_finished",
                json!({"session_id":sid,"item":item_snapshot,"status":status,"lead_notified":notified,"attention_required":!notified}),
            )?;
        }
        tx.execute(
            "UPDATE team_completions SET processed=TRUE WHERE session_id=$1 AND submission_id=$2",
            &[&sid, &key],
        )?;
        tx.commit()?;
        Ok(())
    }
}

pub(crate) fn assignment_finished(tx: &mut Transaction<'_>, item: &Item) -> Result<()> {
    if let (Some(sid), Some(key)) = (&item.assignee, &item.submission_id) {
        let status: String = tx
            .query_opt(
                "SELECT status FROM submissions WHERE session_id=$1 AND id=$2",
                &[sid, key],
            )?
            .context("Assignment receipt missing")?
            .get(0);
        ensure!(terminal(&status),"Previous assignment is still queued or running; cancel/wait before reassigning or accepting");
    }
    Ok(())
}
pub(crate) fn current_assignment(tx: &mut Transaction<'_>, sid: &str, item: &Item) -> Result<()> {
    let s = crate::db::locked(tx, sid, None)?;
    let t = s.turn.as_ref().context("No worker turn")?;
    ensure!(
        item.submission_id.as_deref() == Some(&t.submission_id),
        "This turn does not own the current assignment"
    );
    Ok(())
}
pub(crate) fn session_event(
    tx: &mut Transaction<'_>,
    sid: &str,
    tid: Option<&str>,
    kind: &str,
    data: Value,
) -> Result<()> {
    let row = tx.query_one(
        "UPDATE sessions SET cursor=cursor+1 WHERE id=$1 RETURNING cursor,epoch",
        &[&sid],
    )?;
    let cursor: i64 = row.get(0);
    let epoch: i64 = row.get(1);
    tx.execute(
        "INSERT INTO events VALUES($1,$2,$3,$4,$5)",
        &[&sid, &cursor, &tid, &kind, &data],
    )?;
    tx.execute(
        "INSERT INTO audit(session_id,cursor,turn_id,kind,owner_epoch) VALUES($1,$2,$3,$4,$5)",
        &[&sid, &cursor, &tid, &kind, &epoch],
    )?;
    Ok(())
}
pub(crate) fn deliver(
    tx: &mut Transaction<'_>,
    board: &mut Board,
    sid: &str,
    item: Option<&Item>,
    text: &str,
    queue_limit: u16,
) -> Result<String> {
    ensure!(
        board.deliveries < 128,
        "Team wake budget exhausted; stop and ask the application to review the team"
    );
    let session = crate::db::locked(tx, sid, None)?;
    ensure!(
        !session.closed && !session.provider_blocked,
        "Recipient unavailable or provider-blocked"
    );
    crate::submissions::verify(tx, sid)?;
    let queued: i64 = tx
        .query_one(
            "SELECT count(*) FROM submissions WHERE session_id=$1 AND status='queued'",
            &[&sid],
        )?
        .get(0);
    ensure!(queued < i64::from(queue_limit), "Recipient queue full");
    let key = openworker_runtime::id("delivery");
    let receipt = crate::submissions::insert(
        tx,
        &crate::TurnSubmission {
            limits: None,
            session_id: sid.into(),
            submission_id: key.clone(),
            text: text.into(),
        },
        "queued",
        None,
    )?;
    let item_id = item.map(|i| i.id.as_str());
    let assignment = item.map(|i| i.assignment as i64);
    tx.execute(
        "INSERT INTO team_deliveries VALUES($1,$2,$3,$4,$5)",
        &[&sid, &key, &board.id, &item_id, &assignment],
    )?;
    board.deliveries += 1;
    session_event(
        tx,
        sid,
        Some(&receipt.turn_id),
        "team_delivery_queued",
        json!({"team_id":board.id,"submission_id":key,"turn_id":receipt.turn_id,"item_id":item_id}),
    )?;
    Ok(key)
}
pub(crate) fn notice(tx: &mut Transaction<'_>, sid: &str, key: &str) -> Result<Option<Value>> {
    if !crate::teams::present(tx)? {
        return Ok(None);
    }
    let Some(row)=tx.query_opt("SELECT team_id,item_id,assignment FROM team_deliveries WHERE session_id=$1 AND submission_id=$2",&[&sid,&key])? else {return Ok(None);};
    let team: String = row.get(0);
    let item: Option<String> = row.get(1);
    let assignment: Option<i64> = row.get(2);
    let board = crate::teams::load(tx, &team, false)?;
    ensure!(!board.cancelled, "Team cancelled");
    if let Some(item) = &item {
        let current = board.items.get(item).context("Assignment missing")?;
        ensure!(
            current.assignee.as_deref() == Some(sid)
                && current.assignment as i64 == assignment.unwrap_or(-1)
                && current.submission_id.as_deref() == Some(key)
                && !matches!(current.status, Status::Done | Status::Cancelled),
            "Assignment changed"
        );
        ensure!(
            current
                .dependencies
                .iter()
                .all(|d| board.items[d].status == Status::Done),
            "Dependencies not complete"
        );
    }
    Ok(Some(
        json!({"team_id":team,"item_id":item,"assignment":assignment}),
    ))
}
pub(crate) fn check(tx: &mut Transaction<'_>, s: &Session, kind: &str) -> Result<()> {
    if matches!(
        kind,
        "model_request" | "tool_started" | "interaction_resolved" | "team_tool_finished"
    ) {
        if let Some(turn) = &s.turn {
            notice(tx, &s.id, &turn.submission_id)?;
        }
    }
    Ok(())
}
pub(crate) fn cancel(tx: &mut Transaction<'_>, board: &mut Board, actor: &str) -> Result<()> {
    if board.cancelled {
        return Ok(());
    }
    board.cancelled = true;
    for sid in board.members.keys() {
        let session = crate::db::locked(tx, sid, None)?;
        tx.execute("UPDATE submissions SET status='cancelled' WHERE session_id=$1 AND status='queued' AND id IN (SELECT submission_id FROM team_deliveries WHERE session_id=$1 AND team_id=$2)",&[&sid,&board.id])?;
        if let Some(t) = session.turn.as_ref().filter(|t| !terminal(&t.status)) {
            let command = openworker_runtime::control::Command {
                session_id: sid.clone(),
                request_id: openworker_runtime::id("team_cancel"),
                turn_id: t.id.clone(),
                actor: actor.into(),
                action: openworker_runtime::control::Action::Cancel,
            };
            if tx
                .query_opt(
                    "SELECT id FROM controls WHERE session_id=$1 AND target=$2",
                    &[&sid, &command.target()],
                )?
                .is_none()
            {
                let seq:i64=tx.query_one("UPDATE sessions SET control_count=control_count+1 WHERE id=$1 RETURNING control_count",&[&sid])?.get(0);
                tx.execute("INSERT INTO controls(session_id,id,turn_id,target,command,sequence,status) VALUES($1,$2,$3,$4,$5,$6,'pending')",&[&sid,&command.request_id,&t.id,&command.target(),&json!(command),&seq])?;
                session_event(
                    tx,
                    sid,
                    Some(&t.id),
                    "control_received",
                    json!({"request_id":command.request_id,"target":command.target(),"actor":actor,"status":"pending"}),
                )?;
            }
        }
        session_event(
            tx,
            sid,
            session.turn.as_ref().map(|t| t.id.as_str()),
            "team_cancelled",
            json!({"team_id":board.id}),
        )?;
    }
    Ok(())
}