Python
//! Bounded logical isolation tests, not an OS sandbox or throughput benchmark.
use super::*;
use std::collections::BTreeSet;
use std::sync::{Arc, Barrier};
fn pool(root: &Path, schema: &str, name: &str, sid: &str) -> Child {
Command::new(env!("CARGO_BIN_EXE_pg-worker-fixture"))
.arg(root)
.args([schema, name, sid, "service", "2"])
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::inherit())
.kill_on_drop(true)
.spawn()
.unwrap()
}
pub(super) async fn sessions(
root: &Path,
seeder: &mut Worker,
count: usize,
bypass: bool,
) -> Vec<String> {
let mut ids = vec![];
for n in 0..count {
let workspace = root.join(format!("work{n}"));
std::fs::create_dir(&workspace).unwrap();
let mut params =
json!({"name":format!("concurrent-{n}"),"coworker":"pg-test","workspace":workspace});
if bypass {
params["permissions"] = json!({"mode":"bypass"});
}
let value = seeder.rpc("session/create", params).await;
ids.push(value["session"]["id"].as_str().unwrap().to_owned());
}
ids
}
pub(super) fn effects(root: &Path, n: usize) -> Vec<Value> {
match std::fs::read_to_string(root.join(format!("work{n}/effects.jsonl"))) {
Ok(s) => s
.split_inclusive('\n')
.filter(|l| l.ends_with('\n'))
.map(|l| serde_json::from_str(l).unwrap())
.collect(),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => vec![],
Err(e) => panic!("effect read failed: {e}"),
}
}
pub(super) async fn wait(mut condition: impl FnMut() -> bool) {
tokio::time::timeout(Duration::from_secs(35), async {
while !condition() {
tokio::time::sleep(Duration::from_millis(50)).await;
}
})
.await
.unwrap();
}
pub(super) fn submit(
api: &PostgresStore,
sid: &str,
id: &str,
) -> openworker_store_postgres::SubmissionReceipt {
api.submit_turn(TurnSubmission {
limits: None,
session_id: sid.into(),
submission_id: id.into(),
text: plan("write_gate"),
})
.unwrap()
}
pub(super) fn cancel(api: &PostgresStore, sid: &str, id: &str) -> Control {
let command = Control {
session_id: sid.into(),
request_id: id.into(),
turn_id: api.inspect(sid).unwrap().turn.unwrap().id,
actor: "concurrency-test".into(),
action: Action::Cancel,
};
api.submit_control(command.clone()).unwrap();
command
}
pub(super) fn check_effects(api: &PostgresStore, root: &Path, ids: &[String], expected: &[usize]) {
for (n, sid) in ids.iter().enumerate() {
let records = effects(root, n);
assert_eq!(
records.len(),
expected[n],
"unexpected effect count for {sid}"
);
let mut identities = BTreeSet::new();
for effect in records {
assert_eq!(effect["session_id"], *sid, "cross-session tool context");
assert!(
identities.insert((effect["turn_id"].to_string(), effect["call_id"].to_string())),
"tool effect replayed"
);
assert!(api
.read_events(sid, 0)
.unwrap()
.iter()
.any(|e| e.kind == "tool_started"
&& e.turn_id.as_deref() == effect["turn_id"].as_str()
&& e.data["call_id"] == effect["call_id"]));
}
let events = api.read_events(sid, 0).unwrap();
assert!(
events.len() < 500,
"test must page before exceeding one event page"
);
for (i, e) in events.iter().enumerate() {
assert_eq!(e.session_id, *sid);
assert_eq!(e.event_id, i as i64 + 1);
}
assert_eq!(
events.len() as i64,
api.read_snapshot(sid).unwrap()["cursor"].as_i64().unwrap()
);
}
}
pub(super) fn evidence(config: &Config, name: &str, value: Value) {
if let Some(root) = std::env::var_os("OW_CONCURRENCY_REPORT_DIR") {
let root = std::path::PathBuf::from(root);
assert!(root.is_absolute());
use std::os::unix::fs::DirBuilderExt;
std::fs::DirBuilder::new()
.recursive(true)
.mode(0o700)
.create(&root)
.unwrap();
let path = root.join(format!("{name}-{}.json", config.schema));
use std::io::Write;
use std::os::unix::fs::OpenOptionsExt;
let mut f = std::fs::OpenOptions::new()
.create_new(true)
.write(true)
.mode(0o600)
.open(path)
.unwrap();
writeln!(f, "{value}").unwrap();
}
}
fn effect_evidence(api: &PostgresStore, root: &Path, ids: &[String]) -> Value {
json!(ids.iter().enumerate().map(|(n,sid)|json!({"session_id":sid,"cursor":api.read_snapshot(sid).unwrap()["cursor"],"effects":effects(root,n)})).collect::<Vec<_>>())
}
pub(super) fn remove_owned_schema(config: &Config) {
assert!(
config.schema.starts_with("ow_service_")
&& config.schema[11..].len() == 32
&& config.schema[11..].bytes().all(|c| c.is_ascii_hexdigit())
);
tokio::task::block_in_place(|| {
let mut db = postgres::Client::connect(&config.connection, postgres::NoTls).unwrap();
db.batch_execute(&format!("DROP SCHEMA {} CASCADE", config.schema))
.unwrap();
});
}
async fn multi_slot_fault(database_loss: bool) {
let (root, config, api) = service_fixture();
let mut seeder = Worker::start(root.path(), &config.schema, "seed", None).await;
let ids = sessions(root.path(), &mut seeder, 8, true).await;
// Two independent API-store incarnations submit the same id/text in parallel.
let barrier = Arc::new(Barrier::new(3));
let mut producers = vec![];
for _ in 0..2 {
let c = config.clone();
let ids = ids.clone();
let b = barrier.clone();
producers.push(std::thread::spawn(move || {
let a = PostgresStore::open(c).unwrap();
b.wait();
ids.iter()
.map(|sid| submit(&a, sid, "first").turn_id)
.collect::<Vec<_>>()
}));
}
barrier.wait();
let first = producers.remove(0).join().unwrap();
assert_eq!(first, producers.remove(0).join().unwrap());
seeder.kill().await;
let mut a = pool(root.path(), &config.schema, "a", &ids[0]);
let apid = a.id().unwrap();
wait(|| {
(0..8)
.filter(|&n| !effects(root.path(), n).is_empty())
.count()
== 2
})
.await;
let mut b = pool(root.path(), &config.schema, "b", &ids[0]);
let bpid = b.id().unwrap();
wait(|| {
(0..8)
.filter(|&n| !effects(root.path(), n).is_empty())
.count()
== 4
})
.await;
let dead: Vec<_> = (0..8)
.filter(|&n| {
effects(root.path(), n)
.first()
.is_some_and(|v| v["pid"] == apid)
})
.collect();
let alive: Vec<_> = (0..8)
.filter(|&n| {
effects(root.path(), n)
.first()
.is_some_and(|v| v["pid"] == bpid)
})
.collect();
assert_eq!(dead.len(), 2);
assert_eq!(alive.len(), 2);
// Saturation holds across >2 lease periods. No fifth effect starts and no
// duplicate effects are hidden by overwriting a file.
tokio::time::sleep(Duration::from_secs(13)).await;
assert!(a.try_wait().unwrap().is_none() && b.try_wait().unwrap().is_none());
assert_eq!(
(0..8).map(|n| effects(root.path(), n).len()).sum::<usize>(),
4
);
let follow: Vec<_> = ids
.iter()
.map(|sid| submit(&api, sid, "follow").turn_id)
.collect();
let sibling = alive[1];
let sibling_turn = api.inspect(&ids[sibling]).unwrap().turn.unwrap().id;
let api_b = PostgresStore::open(config.clone()).unwrap();
let cancellation = cancel(&api_b, &ids[alive[0]], "cancel-active");
terminal_receipt(&api, &ids[alive[0]], "first", SubmissionStatus::Interrupted).await;
assert_eq!(
api_b.submit_control(cancellation).unwrap().status,
ControlStatus::Applied
);
assert_eq!(
api.inspect(&ids[sibling]).unwrap().turn.unwrap().id,
sibling_turn
);
assert!(b.try_wait().unwrap().is_none());
if database_loss {
tokio::task::block_in_place(|| {
let mut db = postgres::Client::connect(&config.connection, postgres::NoTls).unwrap();
let owner: String = db
.query_one(
&format!("SELECT owner FROM {}.sessions WHERE id=$1", config.schema),
&[&ids[dead[0]]],
)
.unwrap()
.get(0);
let terminated = db
.query(
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name=$1",
&[&format!("ow-store-{owner}")],
)
.unwrap();
assert_eq!(
terminated.len(),
1,
"fault must target exactly one owned DB connection"
);
});
assert!(!tokio::time::timeout(Duration::from_secs(15), a.wait())
.await
.unwrap()
.unwrap()
.success());
} else {
a.kill().await.unwrap();
}
assert!(b.try_wait().unwrap().is_none(), "unrelated worker died");
assert_eq!(
api.inspect(&ids[sibling]).unwrap().turn.unwrap().id,
sibling_turn
);
assert_eq!(effects(root.path(), sibling).len(), 1);
let mut replacement = pool(root.path(), &config.schema, "replacement", &ids[0]);
// Release all test-controlled barriers; new explicit follow-ups may proceed,
// but crashed/uncertain original effects must never be dispatched again.
for n in 0..8 {
std::fs::write(root.path().join(format!("work{n}/release")), b"release").unwrap();
}
for (n, sid) in ids.iter().enumerate() {
let expected = if dead.contains(&n) || n == alive[0] {
SubmissionStatus::Interrupted
} else {
SubmissionStatus::Completed
};
terminal_receipt(&api, sid, "first", expected).await;
terminal_receipt(&api, sid, "follow", SubmissionStatus::Completed).await;
assert_eq!(submit(&api_b, sid, "first").turn_id, first[n]);
assert_eq!(submit(&api_b, sid, "follow").turn_id, follow[n]);
let snapshot = api.inspect(sid).unwrap();
let unknown = snapshot
.history
.iter()
.filter(|v| {
v["type"] == "function_call_output"
&& v["output"]
.as_str()
.is_some_and(|t| t.contains("tool_outcome_unknown"))
})
.count();
assert_eq!(
unknown,
usize::from(dead.contains(&n) || n == alive[0]),
"unknown result leaked/missing"
);
let lines = effects(root.path(), n);
assert_eq!(lines.len(), 2);
assert_eq!(lines[0]["turn_id"], first[n]);
assert_eq!(lines[1]["turn_id"], follow[n]);
}
stop_service(&mut b).await;
stop_service(&mut replacement).await;
check_effects(&api, root.path(), &ids, &[2; 8]);
evidence(
&config,
if database_loss {
"database-loss"
} else {
"process-death"
},
json!({"sessions":8,"slots_per_worker":2,"workers":2,"effects":16,"duplicate_effects":0,"interrupted_originals":3,"completed_followups":8,"per_session_fifo":true,"sibling_worker_survived":true,"synthetic_native_tools":true,"records":effect_evidence(&api,root.path(),&ids)}),
);
remove_owned_schema(&config);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "requires isolated OW_PG_TEST_URL and native Linux"]
async fn eight_sessions_four_slots_process_death_preserves_fifo_and_siblings() {
multi_slot_fault(false).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "requires isolated OW_PG_TEST_URL and native Linux"]
async fn eight_sessions_four_slots_database_loss_is_scoped_and_not_replayed() {
multi_slot_fault(true).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "requires isolated OW_PG_TEST_URL and native Linux"]
async fn parked_approvals_do_not_consume_slots_or_share_session_grants() {
let (root, config, api) = service_fixture();
let mut seeder = Worker::start(root.path(), &config.schema, "seed", None).await;
let ids = sessions(root.path(), &mut seeder, 6, false).await;
for sid in &ids {
submit(&api, sid, "first");
}
seeder.kill().await;
let mut a = pool(root.path(), &config.schema, "a", &ids[0]);
let mut b = pool(root.path(), &config.schema, "b", &ids[0]);
wait(|| {
ids.iter().all(|sid| {
api.inspect(sid)
.unwrap()
.interactions
.iter()
.any(|i| i.response.is_none())
})
})
.await;
assert_eq!(
(0..6).map(|n| effects(root.path(), n).len()).sum::<usize>(),
0
);
let pending: Vec<_> = ids
.iter()
.map(|sid| api.read_snapshot(sid).unwrap())
.collect();
let api_b = PostgresStore::open(config.clone()).unwrap();
// Use another session's interaction while targeting this session/turn: reject.
let mut wrong = decision(&ids[4], &pending[0], "wrong-session", "once");
wrong.turn_id = pending[4]["turn"]["id"].as_str().unwrap().into();
assert!(api_b.submit_control(wrong).is_err());
let granted = decision(&ids[0], &pending[0], "allow-session", "session");
let aa = api.clone();
let bb = api_b.clone();
let x = granted.clone();
let y = granted.clone();
let (left, right) = tokio::join!(
tokio::task::spawn_blocking(move || aa.submit_control(x).unwrap()),
tokio::task::spawn_blocking(move || bb.submit_control(y).unwrap())
);
assert_eq!(left.unwrap().command, right.unwrap().command);
api_b
.submit_control(decision(&ids[1], &pending[1], "allow-once", "once"))
.unwrap();
api_b
.submit_control(decision(&ids[2], &pending[2], "deny", "deny"))
.unwrap();
cancel(&api_b, &ids[3], "cancel-waiting");
for n in 0..6 {
std::fs::write(root.path().join(format!("work{n}/release")), b"release").unwrap();
}
for sid in &ids[..3] {
terminal_receipt(&api, sid, "first", SubmissionStatus::Completed).await;
}
terminal_receipt(&api, &ids[3], "first", SubmissionStatus::Cancelled).await;
submit(&api_b, &ids[0], "second");
submit(&api_b, &ids[1], "second");
terminal_receipt(&api, &ids[0], "second", SubmissionStatus::Completed).await;
wait(|| {
api.inspect(&ids[1])
.unwrap()
.interactions
.iter()
.any(|i| i.response.is_none())
})
.await;
assert_eq!(api.inspect(&ids[0]).unwrap().approval_grants.len(), 1);
for sid in &ids[1..] {
assert!(api.inspect(sid).unwrap().approval_grants.is_empty());
}
for n in [4, 5] {
assert_eq!(
api.read_snapshot(&ids[n]).unwrap()["interactions"],
pending[n]["interactions"]
);
}
api_b
.submit_control(decision(
&ids[1],
&api.read_snapshot(&ids[1]).unwrap(),
"deny-second",
"deny",
))
.unwrap();
terminal_receipt(&api, &ids[1], "second", SubmissionStatus::Completed).await;
for sid in &ids[4..] {
cancel(&api_b, sid, "cleanup-cancel");
terminal_receipt(&api, sid, "first", SubmissionStatus::Cancelled).await;
}
stop_service(&mut a).await;
stop_service(&mut b).await;
assert_eq!(
api_b.submit_control(granted).unwrap().status,
ControlStatus::Applied
);
check_effects(&api, root.path(), &ids, &[2, 1, 0, 0, 0, 0]);
evidence(
&config,
"approval-isolation",
json!({"sessions":6,"execution_slots":4,"all_six_parked":true,"session_grants":1,"cross_session_grants":0,"duplicate_decision_applied_once":true,"effects":3,"pending_identity_preserved":true,"synthetic_native_tools":true,"records":effect_evidence(&api,root.path(),&ids)}),
);
remove_owned_schema(&config);
}