Skip to content
Python
11.5 KB
Raw Blame
#![cfg(feature = "test-support")]
//! Real PostgreSQL worker pool: a parked lead cannot occupy the child's only slot.
use openworker_runtime::{execution::Backend, model::Model, runtime::Runtime, store::StateStore};
use openworker_store_postgres::{
    worker::{Factory, Options, Selector, Worker},
    Config, PostgresStore, Tls, TurnSubmission,
};
use serde_json::json;
use std::{
    path::{Path, PathBuf},
    sync::Arc,
    time::Duration,
};
use tokio_util::sync::CancellationToken;
fn plan(steps: serde_json::Value) -> String {
    format!("TEST_RESOURCE_GRANTS {steps}")
}
fn runtime(root: &Path, kind: &str, db: PostgresStore) -> Arc<Runtime> {
    Runtime::open_instrumented(
        &root.join(format!("rt-{kind}-{}", uuid::Uuid::new_v4().simple())),
        &root.join(kind),
        &std::env::current_exe().unwrap(),
        Model::new("fixture".into(), true),
        Backend::Off,
        None,
        None,
        Default::default(),
        Some(Box::new(db)),
        Default::default(),
    )
    .unwrap()
}
fn selector(rt: &Runtime) -> Selector {
    Selector {
        coworker: rt.coworker().into(),
        bundle_digest: rt.execution_digest().into(),
        backend: Backend::Off,
        scripted: true,
        fallback_model: "fixture".into(),
        model_provider: Default::default(),
        model_connections: Default::default(),
        environment_provider: None,
        environment_identity: None,
        environment_profile: None,
    }
}
async fn scenario(retire_mode: Option<openworker_runtime::team_lifecycle::Mode>) {
    let tmp = tempfile::tempdir().unwrap();
    let root = tmp.path().canonicalize().unwrap();
    std::fs::create_dir(root.join("work")).unwrap();
    for kind in ["lead", "builder"] {
        std::fs::create_dir_all(root.join(kind).join("coworker")).unwrap();
        std::fs::write(
            root.join(kind).join("coworker/manifest.md"),
            format!(
                "---\nid: {kind}\nname: {kind}\nplanning: true\n---\nCoordinate using named teams."
            ),
        )
        .unwrap();
        std::fs::write(
            root.join(kind).join("tools.json"),
            r#"{"version":3,"tools":[]}"#,
        )
        .unwrap();
    }
    let mut config = Config::new(
        std::env::var("OW_PG_TEST_URL").unwrap(),
        format!("ow_one_slot_{}", uuid::Uuid::new_v4().simple()),
    );
    config.tls = Tls::LoopbackPlaintext;
    PostgresStore::migrate(&config).unwrap();
    let mut seed = PostgresStore::open(config.clone()).unwrap();
    let lead_rt = runtime(&root, "lead", seed.clone());
    let builder_rt = runtime(&root, "builder", seed.clone());
    let lead = lead_rt
        .prepare_session(json!({"coworker":"lead","workspace":root.join("work"),"name":"Lead"}))
        .unwrap();
    seed.save(&lead, "created", json!({})).unwrap();
    let doc = seed
        .agent_team(
            &lead.id,
            "create",
            &json!({"agent_id":"lead","agent_types":["builder"]}),
            None,
        )
        .unwrap();
    let reserved = seed
        .agent_team(
            &lead.id,
            "team_spawn",
            &json!({"agent_type":"builder","agent_id":"builder-ui","message":"Work"}),
            None,
        )
        .unwrap();
    let mut child = builder_rt
        .prepare_session(
            json!({"coworker":"builder","workspace":root.join("work"),"name":"Builder"}),
        )
        .unwrap();
    child.id = reserved["instance"]["session_id"].as_str().unwrap().into();
    child.agent_team = Some(openworker_runtime::agent_team::Membership {
        team_id: doc["id"].as_str().unwrap().into(),
        agent_id: "builder-ui".into(),
        inbox_cursor: 0,
        last_output: None,
        notices: vec![],
        waiting: None,
    });
    seed.save(&child, "child", json!({})).unwrap();
    seed.agent_team(&lead.id, "ready", &json!({"agent_id":"builder-ui"}), None)
        .unwrap();
    let selectors = [selector(&lead_rt), selector(&builder_rt)];
    // These seed runtimes never execute; release before a one-slot pool takes over.
    seed.release(&lead.id).unwrap();
    seed.release(&child.id).unwrap();
    drop(lead_rt);
    drop(builder_rt);
    let mut factories = vec![];
    for kind in ["lead", "builder"] {
        let root: PathBuf = root.clone();
        let kind = kind.to_owned();
        let factory: Factory = Arc::new(move |_, db| Ok(runtime(&root, &kind, db)));
        factories.push(factory);
    }
    let catalog: Vec<_> = selectors
        .iter()
        .cloned()
        .zip(factories.iter().cloned())
        .collect();
    let stop = CancellationToken::new();
    let worker = Worker::new(
        config.clone(),
        selectors[0].clone(),
        Options {
            slots: 1,
            poll_interval: Duration::from_millis(50),
            ..Default::default()
        },
        factories[0].clone(),
    )
    .unwrap()
    .with_catalog(Arc::new(move || Ok(catalog.clone())));
    seed.submit_turn(TurnSubmission{session_id:lead.id.clone(),submission_id:"lead-task".into(),text:plan(json!([{"name":"team_wait_agent","arguments":{"agent_ids":["builder-ui"],"timeout_ms":60000}}])),limits:None}).unwrap();

    let token = stop.clone();
    let task = tokio::spawn(async move {
        let result = worker.run(token).await;
        if let Err(error) = &result {
            eprintln!("isolated worker failed: {error:#}");
        }
        result
    });
    // Establish the test's ordering, rather than assuming a fair scheduler will
    // choose the parent first when both sessions already have queued work.
    tokio::time::timeout(Duration::from_secs(30), async {
        loop {
            if seed.read_snapshot(&lead.id).unwrap()["turn"]["status"] == "waiting_agents" {
                break;
            }
            assert!(!task.is_finished());
            tokio::time::sleep(Duration::from_millis(50)).await;
        }
    })
    .await
    .expect("Lead must park before child admission");
    let child_steps = if let Some(mode) = retire_mode {
        json!([{"name":"team_wait_agent","arguments":{"agent_ids":["lead"],"timeout_ms":if mode==openworker_runtime::team_lifecycle::Mode::Cancel {60000}else{1500}}},{"name":"team_post_board","arguments":{"message":"@lead verified child proof"}}])
    } else {
        json!([{"name":"team_post_board","arguments":{"message":"@lead verified child proof"}}])
    };
    seed.submit_turn(TurnSubmission {
        session_id: child.id.clone(),
        submission_id: "child-task".into(),
        text: plan(child_steps),
        limits: None,
    })
    .unwrap();
    if let Some(mode) = retire_mode {
        tokio::time::timeout(Duration::from_secs(30), async {
            loop {
                if seed.read_snapshot(&child.id).unwrap()["turn"]["status"] == "waiting_agents" {
                    break;
                }
                assert!(!task.is_finished());
                tokio::time::sleep(Duration::from_millis(25)).await;
            }
        })
        .await
        .expect("Child must park before retirement");
        seed.submit_turn(TurnSubmission {
            session_id: child.id.clone(),
            submission_id: "queued-later".into(),
            text: "Never start after retirement".into(),
            limits: None,
        })
        .unwrap();
        seed.agent_team_api(&lead.id,"command",json!({"request_id":"retire-child","operation":"team_manage","arguments":{"action":"retire","agent_id":"builder-ui","mode":mode}})).unwrap();
        tokio::time::timeout(Duration::from_secs(10), async {
            loop {
                seed.reconcile_stopped_agent_team().unwrap();
                let status = seed.submission(&child.id, "queued-later").unwrap().status;
                assert_ne!(
                    status,
                    openworker_store_postgres::SubmissionStatus::Started,
                    "Retirement cannot promote queued followups"
                );
                if status == openworker_store_postgres::SubmissionStatus::Cancelled {
                    break;
                }
                tokio::time::sleep(Duration::from_millis(25)).await;
            }
        })
        .await
        .expect("Controller must reconcile even when a worker temporarily holds the row lock");
        assert!(seed
            .submit_turn(TurnSubmission {
                session_id: child.id.clone(),
                submission_id: "too-late".into(),
                text: "Blocked".into(),
                limits: None
            })
            .is_err());
        tokio::time::timeout(Duration::from_secs(30), async {
            loop {
                seed.reconcile_stopped_agent_team().unwrap();
                let s = seed.read_snapshot(&child.id).unwrap();
                let doc = seed
                    .agent_team_api(&lead.id, "snapshot", json!({}))
                    .unwrap();
                if s["session"]["closed"] == true
                    && doc["roster"]["instances"]["builder-ui"]["phase"] == "closed"
                {
                    break;
                }
                assert!(!task.is_finished());
                tokio::time::sleep(Duration::from_millis(50)).await;
            }
        })
        .await
        .expect("Owning worker must settle retirement");
        assert_eq!(
            seed.submission(&child.id, "queued-later").unwrap().status,
            openworker_store_postgres::SubmissionStatus::Cancelled
        );
        assert_eq!(
            seed.read_snapshot(&child.id).unwrap()["turn"]["status"],
            if mode == openworker_runtime::team_lifecycle::Mode::Cancel {
                "cancelled"
            } else {
                "completed"
            }
        );
    }

    let result = tokio::time::timeout(Duration::from_secs(45), async {
        loop {
            let s = seed.read_snapshot(&lead.id).unwrap();
            if s["turn"]["status"] == "completed" {
                break s;
            }
            assert!(!task.is_finished(), "Worker must remain healthy");
            tokio::time::sleep(Duration::from_millis(50)).await;
        }
    })
    .await;
    stop.cancel();
    task.await.unwrap().unwrap();
    let result = result.expect("A parent must not starve its child");
    assert_eq!(result["turn"]["status"], "completed");
    let events = seed.read_events(&lead.id, 0).unwrap();
    assert!(events
        .iter()
        .any(|e| e.kind == "agent_team" && e.data["operation"] == "team_wait_agent"));
    assert!(
        events.iter().filter(|e| e.kind == "owner_claimed").count() >= 2,
        "Lead must release and reacquire ownership"
    );
    if retire_mode != Some(openworker_runtime::team_lifecycle::Mode::Cancel) {
        assert!(result["messages"]
            .to_string()
            .contains("verified child proof"));
    }
    assert!(!result["messages"]
        .to_string()
        .contains("\\\"timeout\\\":true"));
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "requires isolated OW_PG_TEST_URL"]
async fn one_slot_wait_releases_and_resumes_across_worker_ownership() {
    scenario(None).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "requires isolated OW_PG_TEST_URL"]
async fn finish_current_retirement_resumes_parked_work_and_cancels_queued_followups() {
    scenario(Some(
        openworker_runtime::team_lifecycle::Mode::FinishCurrent,
    ))
    .await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "requires isolated OW_PG_TEST_URL"]
async fn cancel_retirement_settles_parked_work_and_releases_its_slot() {
    scenario(Some(openworker_runtime::team_lifecycle::Mode::Cancel)).await;
}