Python
#![cfg(all(feature = "test-support", target_os = "linux"))]
#[path = "support/concurrency.rs"]
mod concurrency;
#[path = "support/draining.rs"]
mod draining;
#[path = "support/load.rs"]
mod load;
use openworker_runtime::control::{Action, Binding, Command as Control, Status as ControlStatus};
use openworker_store_postgres::{Config, PostgresStore, SubmissionStatus, Tls, TurnSubmission};
use serde_json::{json, Value};
use std::{path::Path, process::Stdio, time::Duration};
use tokio::{
io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
process::{Child, ChildStdin, ChildStdout, Command},
};
struct Worker {
child: Child,
input: Option<ChildStdin>,
output: BufReader<ChildStdout>,
id: u64,
}
fn service(root: &Path, schema: &str, name: &str, sid: &str) -> Child {
Command::new(env!("CARGO_BIN_EXE_pg-worker-fixture"))
.arg(root)
.args([schema, name, sid, "service"])
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::inherit())
.kill_on_drop(true)
.spawn()
.unwrap()
}
async fn stop_service(child: &mut Child) {
// Signal only our still-owned child, then wait for bounded graceful shutdown.
assert!(child.try_wait().unwrap().is_none());
assert!(Command::new("kill")
.args(["-INT", &child.id().unwrap().to_string()])
.status()
.await
.unwrap()
.success());
assert!(tokio::time::timeout(Duration::from_secs(15), child.wait())
.await
.unwrap()
.unwrap()
.success());
}
fn plan(action: &str) -> String {
format!(
"TEST_RESOURCE_GRANTS {}",
json!([{"name":"test__probe","arguments":{"action":action}}])
)
}
fn service_fixture() -> (tempfile::TempDir, Config, PostgresStore) {
let root = tempfile::tempdir().unwrap();
std::fs::create_dir_all(root.path().join("bundle/coworker")).unwrap();
std::fs::write(
root.path().join("bundle/coworker/manifest.md"),
"---\nid: pg-test\nname: Service test\n---\nUse the tool.",
)
.unwrap();
std::fs::write(
root.path().join("bundle/tools.json"),
r#"{"version":3,"tools":["test.probe"]}"#,
)
.unwrap();
let mut config = Config::new(
std::env::var("OW_PG_TEST_URL").unwrap(),
format!("ow_service_{}", uuid::Uuid::new_v4().simple()),
);
config.tls = Tls::LoopbackPlaintext;
config.lease_duration = Duration::from_secs(6);
config.timeout = Duration::from_secs(1);
PostgresStore::migrate(&config).unwrap();
let api = PostgresStore::open(config.clone()).unwrap();
(root, config, api)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[ignore = "requires isolated OW_PG_TEST_URL and native Linux"]
async fn production_worker_plans_survive_replacement_and_cross_api_approval() {
let (root, config, api) = service_fixture();
std::fs::write(
root.path().join("bundle/coworker/manifest.md"),
"---\nid: pg-test\nname: Planner test\nplanning: true\n---\nPlan then use the tool.",
)
.unwrap();
std::fs::create_dir(root.path().join("work")).unwrap();
let other_api = PostgresStore::open(config.clone()).unwrap();
let mut seeder = Worker::start(root.path(), &config.schema, "seed", None).await;
let created = seeder
.rpc(
"session/create",
json!({"name":"Plan test","coworker":"pg-test","workspace":root.path().join("work")}),
)
.await;
let sid = created["session"]["id"].as_str().unwrap();
api.submit_turn(TurnSubmission {
limits: None,
session_id: sid.into(), submission_id: "planned-task".into(),
text: format!("TEST_RESOURCE_GRANTS {}", json!([
{"name":"update_plan","arguments":{"expected_revision":0,"explanation":"Inspect first","steps":[{"id":"inspect","description":"Write one test marker","status":"in_progress"}]}},
{"name":"test__probe","arguments":{"action":"write_once"}},
{"name":"update_plan","arguments":{"expected_revision":1,"explanation":"Marker observed","steps":[{"id":"inspect","description":"Write one test marker","status":"completed"}]}}
])),
}).unwrap();
seeder.kill().await;
let mut a = service(root.path(), &config.schema, "a", sid);
let pending = tokio::time::timeout(Duration::from_secs(20), async {
loop {
let snap = other_api.read_snapshot(sid).unwrap();
if !snap["interactions"].as_array().unwrap().is_empty() {
break snap;
}
assert!(a.try_wait().unwrap().is_none());
tokio::time::sleep(Duration::from_millis(50)).await;
}
})
.await
.unwrap();
assert_eq!(pending["plan"]["revision"], 1);
assert_eq!(pending["interactions"][0]["call"]["name"], "test__probe");
assert!(!root.path().join("work/proof.txt").exists());
stop_service(&mut a).await;
assert_eq!(
other_api.read_snapshot(sid).unwrap()["plan"],
pending["plan"]
);
other_api
.submit_control(decision(sid, &pending, "approve-marker", "once"))
.unwrap();
let mut b = service(root.path(), &config.schema, "b", sid);
terminal_receipt(&api, sid, "planned-task", SubmissionStatus::Completed).await;
stop_service(&mut b).await;
let final_snapshot = other_api.read_snapshot(sid).unwrap();
assert_eq!(final_snapshot["plan"]["revision"], 2);
assert_eq!(final_snapshot["plan"]["steps"][0]["status"], "completed");
let saved = api.inspect(sid).unwrap();
assert!(saved.approval_grants.is_empty());
assert_eq!(saved.interactions.len(), 1);
let events = other_api.read_events(sid, 0).unwrap();
assert_eq!(
events.iter().filter(|e| e.kind == "plan_updated").count(),
2
);
assert_eq!(
std::fs::read_to_string(root.path().join("work/proof.txt"))
.unwrap()
.lines()
.count(),
1
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[ignore = "requires isolated OW_PG_TEST_URL and native Linux"]
async fn production_workers_bound_capacity_renew_and_shutdown_without_replaying_effects() {
let (root, config, api) = service_fixture();
let mut seeder = Worker::start(root.path(), &config.schema, "seed", None).await;
let mut sessions = vec![];
for i in 0..3 {
let workspace = root.path().join(format!("work{i}"));
std::fs::create_dir(&workspace).unwrap();
let created=seeder.rpc("session/create",json!({"name":format!("session{i}"),"coworker":"pg-test","workspace":workspace,"permissions":{"mode":"bypass"}})).await;
let sid = created["session"]["id"].as_str().unwrap().to_owned();
api.submit_turn(TurnSubmission {
limits: None,
session_id: sid.clone(),
submission_id: "effect".into(),
text: plan("write_wait"),
})
.unwrap();
sessions.push(sid);
}
seeder.kill().await;
let mut a = service(root.path(), &config.schema, "a", &sessions[0]);
let mut b = service(root.path(), &config.schema, "b", &sessions[0]);
let files = || {
(0..3)
.filter(|i| root.path().join(format!("work{i}/proof.txt")).exists())
.count()
};
tokio::time::timeout(Duration::from_secs(25), async {
while files() != 2 {
assert!(a.try_wait().unwrap().is_none());
assert!(b.try_wait().unwrap().is_none());
tokio::time::sleep(Duration::from_millis(100)).await;
}
})
.await
.unwrap();
// More than two lease periods. A nonrenewing worker would be stolen here.
tokio::time::sleep(Duration::from_secs(13)).await;
assert_eq!(
files(),
2,
"third session bypassed configured worker capacity"
);
let active = sessions
.iter()
.find(|sid| api.inspect(sid).unwrap().turn.is_some())
.unwrap();
let turn = api.inspect(active).unwrap().turn.unwrap();
api.submit_control(Control {
session_id: active.clone(),
request_id: "cancel".into(),
turn_id: turn.id,
actor: "operator".into(),
action: Action::Cancel,
})
.unwrap();
tokio::time::timeout(Duration::from_secs(15), async {
while files() != 3 {
tokio::time::sleep(Duration::from_millis(100)).await;
}
})
.await
.unwrap();
stop_service(&mut a).await;
stop_service(&mut b).await;
for (i, sid) in sessions.iter().enumerate() {
let saved = api.inspect(sid).unwrap();
assert_eq!(saved.turn.unwrap().status, "interrupted");
assert_eq!(
std::fs::read_to_string(root.path().join(format!("work{i}/proof.txt")))
.unwrap()
.lines()
.count(),
1
);
api.submit_turn(TurnSubmission {
limits: None,
session_id: sid.clone(),
submission_id: "inspect".into(),
text: plan("inspect"),
})
.unwrap();
}
let mut c = service(root.path(), &config.schema, "c", &sessions[0]);
for sid in &sessions {
terminal_receipt(&api, sid, "inspect", SubmissionStatus::Completed).await;
}
stop_service(&mut c).await;
for i in 0..3 {
assert_eq!(
std::fs::read_to_string(root.path().join(format!("work{i}/proof.txt")))
.unwrap()
.lines()
.count(),
1
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[ignore = "requires isolated OW_PG_TEST_URL and native Linux"]
async fn production_worker_loses_database_and_replacement_recovers_unknown_once() {
let (root, config, api) = service_fixture();
std::fs::create_dir(root.path().join("work")).unwrap();
let mut seeder = Worker::start(root.path(), &config.schema, "seed", None).await;
let created=seeder.rpc("session/create",json!({"name":"one","coworker":"pg-test","workspace":root.path().join("work"),"permissions":{"mode":"bypass"}})).await;
let sid = created["session"]["id"].as_str().unwrap();
api.submit_turn(TurnSubmission {
limits: None,
session_id: sid.into(),
submission_id: "effect".into(),
text: plan("write_wait"),
})
.unwrap();
seeder.kill().await;
let mut a = service(root.path(), &config.schema, "a", sid);
tokio::time::timeout(Duration::from_secs(20), async {
while std::fs::read_to_string(root.path().join("work/proof.txt"))
.ok()
.as_deref()
!= Some("one real side effect\n")
{
assert!(a.try_wait().unwrap().is_none());
tokio::time::sleep(Duration::from_millis(100)).await;
}
})
.await
.unwrap();
tokio::task::block_in_place(|| {
let mut db = postgres::Client::connect(&config.connection, postgres::NoTls).unwrap();
db.batch_execute(&format!(
"SET ow.runtime_schema='7'; SET search_path={},pg_catalog",
config.schema
))
.unwrap();
let owner: String = db
.query_one("SELECT owner FROM sessions WHERE id=$1", &[&sid])
.unwrap()
.get(0);
let app = format!("ow-store-{owner}");
let rows = db
.query(
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name=$1",
&[&app],
)
.unwrap();
assert_eq!(rows.len(), 1);
});
assert!(!tokio::time::timeout(Duration::from_secs(15), a.wait())
.await
.unwrap()
.unwrap()
.success());
let mut b = service(root.path(), &config.schema, "b", sid);
terminal_receipt(&api, sid, "effect", SubmissionStatus::Interrupted).await;
assert_eq!(
api.inspect(sid)
.unwrap()
.history
.iter()
.filter(|v| v["type"] == "function_call_output"
&& v.to_string().contains("harness_recovery"))
.count(),
1
);
api.submit_turn(TurnSubmission {
limits: None,
session_id: sid.into(),
submission_id: "inspect".into(),
text: plan("inspect"),
})
.unwrap();
terminal_receipt(&api, sid, "inspect", SubmissionStatus::Completed).await;
stop_service(&mut b).await;
assert_eq!(
std::fs::read_to_string(root.path().join("work/proof.txt"))
.unwrap()
.lines()
.count(),
1
);
}
fn decision(sid: &str, pending: &Value, id: &str, response: &str) -> Control {
let i = &pending["interactions"][0];
Control {
session_id: sid.into(),
request_id: id.into(),
turn_id: i["turn_id"].as_str().unwrap().into(),
actor: "principal-example".into(),
action: Action::Response {
interaction_id: i["id"].as_str().unwrap().into(),
response: response.into(),
binding: Binding {
bundle_digest: i["bundle_digest"].as_str().unwrap().into(),
schema_digest: i["schema_digest"].as_str().unwrap().into(),
policy_digest: i["policy_digest"].as_str().unwrap().into(),
resource_policy_digest: i["resource_policy_digest"].as_str().unwrap().into(),
},
},
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[ignore = "requires isolated OW_PG_TEST_URL and native Linux"]
async fn production_worker_parks_approval_reclaims_for_answer_and_keeps_session_grant() {
let (root, config, api) = service_fixture();
std::fs::create_dir(root.path().join("work")).unwrap();
let mut seeder = Worker::start(root.path(), &config.schema, "seed", None).await;
let created = seeder
.rpc(
"session/create",
json!({"name":"approval","coworker":"pg-test","workspace":root.path().join("work")}),
)
.await;
let sid = created["session"]["id"].as_str().unwrap();
api.submit_turn(TurnSubmission {
limits: None,
session_id: sid.into(),
submission_id: "one".into(),
text: plan("write_once"),
})
.unwrap();
seeder.kill().await;
let mut a = service(root.path(), &config.schema, "a", sid);
let pending = tokio::time::timeout(Duration::from_secs(20), async {
loop {
let s = api.inspect(sid).unwrap();
if s.interactions.iter().any(|i| i.response.is_none()) {
break json!({"interactions":s.interactions});
}
assert!(a.try_wait().unwrap().is_none());
tokio::time::sleep(Duration::from_millis(100)).await;
}
})
.await
.unwrap();
assert!(!root.path().join("work/proof.txt").exists());
let first_observation = api.inspect(sid).unwrap().turn.unwrap().observation.unwrap();
assert_eq!(first_observation.segments, 1);
stop_service(&mut a).await;
let decision = decision(sid, &pending, "allow-session", "session");
api.submit_control(decision.clone()).unwrap();
let mut b = service(root.path(), &config.schema, "b", sid);
terminal_receipt(&api, sid, "one", SubmissionStatus::Completed).await;
let after_resume = api.inspect(sid).unwrap();
let resumed = after_resume
.turn
.as_ref()
.unwrap()
.observation
.as_ref()
.unwrap();
assert_eq!(resumed.segments, 2);
assert_eq!(
resumed.first_started_at_ms,
first_observation.first_started_at_ms
);
assert_eq!(
resumed.last_segment.trace,
first_observation.last_segment.trace
);
assert_ne!(
resumed.last_segment.span,
first_observation.last_segment.span
);
let interaction = &after_resume.interactions[0];
assert!(interaction.requested_at_ms.is_some());
assert!(interaction.resolved_at_ms >= interaction.requested_at_ms);
assert_eq!(
api.control_receipt(sid, "allow-session").unwrap().status,
ControlStatus::Applied
);
api.submit_turn(TurnSubmission {
limits: None,
session_id: sid.into(),
submission_id: "two".into(),
text: plan("write_once"),
})
.unwrap();
terminal_receipt(&api, sid, "two", SubmissionStatus::Completed).await;
stop_service(&mut b).await;
let records: Vec<Value> = std::fs::read_to_string(root.path().join("observations.jsonl"))
.unwrap()
.lines()
.map(|s| serde_json::from_str(s).unwrap())
.collect();
assert!(records.iter().any(|r| r["operation"] == "agent.execute"
&& r["links"]
.as_array()
.is_some_and(|links| links.contains(&json!(first_observation.last_segment)))));
assert_eq!(
records
.iter()
.filter(|r| r["operation"] == "approval.wait" && r["timing_known"] == true)
.count(),
1
);
assert_eq!(
std::fs::read_to_string(root.path().join("work/proof.txt"))
.unwrap()
.lines()
.count(),
2
);
assert_eq!(api.inspect(sid).unwrap().approval_grants.len(), 1);
assert_eq!(
api.submit_control(decision).unwrap().status,
ControlStatus::Applied
);
}
async fn terminal_receipt(api: &PostgresStore, sid: &str, id: &str, status: SubmissionStatus) {
tokio::time::timeout(Duration::from_secs(25), async {
loop {
let receipt = api.submission(sid, id).unwrap();
if receipt.status == status {
break;
}
assert!(
!matches!(
receipt.status,
SubmissionStatus::Failed
| SubmissionStatus::Cancelled
| SubmissionStatus::Truncated
),
"unexpected terminal receipt {receipt:?}"
);
tokio::time::sleep(Duration::from_millis(30)).await;
}
})
.await
.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[ignore = "requires isolated OW_PG_TEST_URL and native Linux"]
async fn remote_controls_survive_crash_grant_session_access_and_cancel_unknown_effect() {
let root = tempfile::tempdir().unwrap();
std::fs::create_dir_all(root.path().join("bundle/coworker")).unwrap();
std::fs::create_dir(root.path().join("work")).unwrap();
std::fs::create_dir(root.path().join("work2")).unwrap();
std::fs::write(
root.path().join("bundle/coworker/manifest.md"),
"---\nid: pg-test\nname: Control test\n---\nUse the test tool.",
)
.unwrap();
std::fs::write(
root.path().join("bundle/tools.json"),
r#"{"version":3,"tools":["test.probe"]}"#,
)
.unwrap();
let schema = format!("ow_remote_{}", uuid::Uuid::new_v4().simple());
let mut config = Config::new(std::env::var("OW_PG_TEST_URL").unwrap(), schema.clone());
config.tls = Tls::LoopbackPlaintext;
config.lease_duration = Duration::from_secs(6);
config.timeout = Duration::from_secs(1);
PostgresStore::migrate(&config).unwrap();
let api_a = PostgresStore::open(config.clone()).unwrap();
let api_b = PostgresStore::open(config).unwrap();
let mut a = Worker::start(root.path(), &schema, "a", None).await;
let created = a
.rpc(
"session/create",
json!({"name":"Control","coworker":"pg-test","workspace":root.path().join("work")}),
)
.await;
let sid = created["session"]["id"].as_str().unwrap();
let plan = |action: &str| {
format!(
"TEST_RESOURCE_GRANTS {}",
json!([{"name":"test__probe","arguments":{"action":action}}])
)
};
a.rpc(
"turn/start",
json!({"session_id":sid,"submission_id":"one","text":plan("write_once")}),
)
.await;
let pending = a.pending(sid).await;
a.kill().await;
let grant = decision(sid, &pending, "session-grant", "session");
assert_eq!(
api_a.submit_control(grant.clone()).unwrap().status,
ControlStatus::Pending
);
assert_eq!(
api_b.submit_control(grant.clone()).unwrap().status,
ControlStatus::Pending
);
tokio::time::sleep(Duration::from_millis(6200)).await;
let mut b = Worker::start_mode(root.path(), &schema, "b", Some(sid), true).await;
terminal_receipt(&api_b, sid, "one", SubmissionStatus::Completed).await;
assert_eq!(
api_b.control_receipt(sid, "session-grant").unwrap().status,
ControlStatus::Applied
);
assert_eq!(api_b.inspect(sid).unwrap().approval_grants.len(), 1);
assert_eq!(
api_b.submit_control(grant).unwrap().status,
ControlStatus::Applied
);
// Same tool/action group reuses the persisted session grant, without a second prompt.
let run = api_a
.submit_turn(TurnSubmission {
limits: None,
session_id: sid.into(),
submission_id: "effect".into(),
text: plan("write_wait"),
})
.unwrap();
tokio::time::timeout(Duration::from_secs(20), async {
loop {
if std::fs::read_to_string(root.path().join("work/proof.txt"))
.unwrap()
.lines()
.count()
== 2
{
break;
}
tokio::time::sleep(Duration::from_millis(30)).await;
}
})
.await
.unwrap();
let cancel = Control {
session_id: sid.into(),
request_id: "cancel-effect".into(),
turn_id: run.turn_id,
actor: "principal-example".into(),
action: Action::Cancel,
};
api_b.submit_control(cancel.clone()).unwrap();
terminal_receipt(&api_a, sid, "effect", SubmissionStatus::Interrupted).await;
assert_eq!(
api_a.control_receipt(sid, "cancel-effect").unwrap().status,
ControlStatus::Applied
);
assert_eq!(
api_a.submit_control(cancel).unwrap().status,
ControlStatus::Applied
);
let history = api_a.inspect(sid).unwrap().history;
assert_eq!(
history
.iter()
.filter(|v| v["type"] == "function_call_output"
&& v["output"]
.as_str()
.is_some_and(|s| s.contains("harness_cancellation")
&& s.contains("tool_outcome_unknown")))
.count(),
1
);
assert_eq!(
std::fs::read_to_string(root.path().join("work/proof.txt"))
.unwrap()
.lines()
.count(),
2
);
// A free-standing question is not a reusable tool approval.
api_a.submit_turn(TurnSubmission{
limits: None,session_id:sid.into(),submission_id:"question".into(),text:format!("TEST_RESOURCE_GRANTS {}",json!([{"name":"ask_user","arguments":{"question":"Continue?","options":["Yes","No"]}}]))}).unwrap();
let pending = b.pending(sid).await;
assert!(api_b
.submit_control(decision(sid, &pending, "answer", "Maybe"))
.is_err());
api_b
.submit_control(decision(sid, &pending, "answer", "Yes"))
.unwrap();
terminal_receipt(&api_a, sid, "question", SubmissionStatus::Completed).await;
assert_eq!(api_a.inspect(sid).unwrap().approval_grants.len(), 1);
// Allow once on a different session must not mint a grant there.
let second = b
.rpc(
"session/create",
json!({"name":"Once","coworker":"pg-test","workspace":root.path().join("work2")}),
)
.await;
let sid2 = second["session"]["id"].as_str().unwrap();
api_a
.submit_turn(TurnSubmission {
limits: None,
session_id: sid2.into(),
submission_id: "once".into(),
text: plan("write_once"),
})
.unwrap();
let pending = b.pending(sid2).await;
api_b
.submit_control(decision(sid2, &pending, "once-decision", "once"))
.unwrap();
terminal_receipt(&api_a, sid2, "once", SubmissionStatus::Completed).await;
assert!(api_a.inspect(sid2).unwrap().approval_grants.is_empty());
api_a
.submit_turn(TurnSubmission {
limits: None,
session_id: sid2.into(),
submission_id: "denied".into(),
text: plan("write_once"),
})
.unwrap();
let pending = b.pending(sid2).await;
api_b
.submit_control(decision(sid2, &pending, "deny-decision", "deny"))
.unwrap();
terminal_receipt(&api_a, sid2, "denied", SubmissionStatus::Completed).await;
// Cancellation while still awaiting approval has a known no-dispatch result.
let queued = api_a
.submit_turn(TurnSubmission {
limits: None,
session_id: sid2.into(),
submission_id: "before-dispatch".into(),
text: plan("write_once"),
})
.unwrap();
b.pending(sid2).await;
api_b
.submit_control(Control {
session_id: sid2.into(),
request_id: "cancel-before".into(),
turn_id: queued.turn_id,
actor: "principal-example".into(),
action: Action::Cancel,
})
.unwrap();
terminal_receipt(&api_a, sid2, "before-dispatch", SubmissionStatus::Cancelled).await;
assert_eq!(
std::fs::read_to_string(root.path().join("work2/proof.txt"))
.unwrap()
.lines()
.count(),
1
);
b.close().await;
}
impl Worker {
async fn start(root: &Path, schema: &str, name: &str, sid: Option<&str>) -> Self {
Self::start_mode(root, schema, name, sid, false).await
}
async fn start_mode(
root: &Path,
schema: &str,
name: &str,
sid: Option<&str>,
consume: bool,
) -> Self {
Self::start_binary(
Path::new(env!("CARGO_BIN_EXE_pg-worker-fixture")),
root,
schema,
name,
sid,
consume,
)
.await
}
async fn start_binary(
program: &Path,
root: &Path,
schema: &str,
name: &str,
sid: Option<&str>,
consume: bool,
) -> Self {
let mut command = Command::new(program);
command.arg(root).arg(schema).arg(name);
if let Some(sid) = sid {
command.arg(sid);
if consume {
command.arg("consume");
}
}
let mut child = command
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::inherit())
.kill_on_drop(true)
.spawn()
.unwrap();
let mut w = Self {
input: child.stdin.take(),
output: BufReader::new(child.stdout.take().unwrap()),
child,
id: 0,
};
w.rpc("initialize", json!({"protocol_version":"0.1"})).await;
w
}
async fn rpc(&mut self, method: &str, params: Value) -> Value {
self.id += 1;
let id = self.id;
let frame = format!(
"{}\n",
json!({"jsonrpc":"2.0","id":id,"method":method,"params":params})
);
tokio::time::timeout(Duration::from_secs(30), async {
loop {
self.input
.as_mut()
.unwrap()
.write_all(frame.as_bytes())
.await
.unwrap();
loop {
let mut line = String::new();
assert!(
self.output.read_line(&mut line).await.unwrap() > 0,
"worker exited"
);
let v: Value = serde_json::from_str(&line).unwrap();
if v["id"] != id {
continue;
}
// Explicit pre-admission busy rejection of an idempotent
// interaction answer only. Never retry tools, lost replies
// or arbitrary errors. Preserve the exact answer/identity.
if method == "interaction/respond"
&& v["error"]["message"]
== "Interaction is checkpointing; retry this same response"
{
tokio::time::sleep(Duration::from_millis(20)).await;
break;
}
assert!(v.get("error").is_none(), "{v}");
return v["result"].clone();
}
}
})
.await
.unwrap()
}
async fn pending(&mut self, sid: &str) -> Value {
tokio::time::timeout(Duration::from_secs(20), async {
loop {
let v = self.rpc("session/attach", json!({"session_id":sid})).await;
if !v["interactions"].as_array().unwrap().is_empty() {
return v;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.unwrap()
}
async fn kill(&mut self) {
self.child.kill().await.unwrap();
self.child.wait().await.unwrap();
}
async fn close(mut self) {
self.input.take();
tokio::time::timeout(Duration::from_secs(10), self.child.wait())
.await
.unwrap()
.unwrap();
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[ignore = "requires isolated OW_PG_TEST_URL and native Linux"]
async fn api_accepts_while_worker_is_dead_and_replacement_consumes_fifo() {
let root = tempfile::tempdir().unwrap();
std::fs::create_dir_all(root.path().join("bundle/coworker")).unwrap();
std::fs::create_dir(root.path().join("work")).unwrap();
std::fs::write(
root.path().join("bundle/coworker/manifest.md"),
"---\nid: pg-test\nname: Queue test\n---\nUse the test tool.",
)
.unwrap();
std::fs::write(
root.path().join("bundle/tools.json"),
r#"{"version":3,"tools":["test.probe"]}"#,
)
.unwrap();
let schema = format!("ow_ingress_{}", uuid::Uuid::new_v4().simple());
let mut config = Config::new(std::env::var("OW_PG_TEST_URL").unwrap(), schema.clone());
config.tls = Tls::LoopbackPlaintext;
config.lease_duration = Duration::from_secs(6);
config.timeout = Duration::from_secs(1);
PostgresStore::migrate(&config).unwrap();
let mut original = Worker::start(root.path(), &schema, "original", None).await;
let created = original
.rpc(
"session/create",
json!({"name":"Queue","coworker":"pg-test","workspace":root.path().join("work")}),
)
.await;
let sid = created["session"]["id"].as_str().unwrap();
original.kill().await;
let api_a = PostgresStore::open(config.clone()).unwrap();
let api_b = PostgresStore::open(config.clone()).unwrap();
let make = |id: &str, action: &str| TurnSubmission {
limits: None,
session_id: sid.into(),
submission_id: id.into(),
text: format!(
"TEST_RESOURCE_GRANTS {}",
json!([{"name":"test__probe","arguments":{"action":action}}])
),
};
let first = api_a.submit_turn(make("one", "write_once")).unwrap();
let second = api_b.submit_turn(make("two", "inspect")).unwrap();
assert_eq!(api_b.submit_turn(make("one", "write_once")).unwrap(), first);
assert_eq!(first.status, SubmissionStatus::Queued);
assert!(api_b.inspect(sid).unwrap().history.is_empty());
tokio::time::sleep(Duration::from_millis(6200)).await;
let mut replacement =
Worker::start_mode(root.path(), &schema, "replacement", Some(sid), true).await;
let pending = replacement.pending(sid).await;
assert_eq!(pending["turn"]["id"], first.turn_id);
assert_eq!(
api_a.submission(sid, "two").unwrap().status,
SubmissionStatus::Queued
);
let iid = &pending["interactions"][0]["id"];
replacement
.rpc(
"interaction/respond",
json!({"session_id":sid,"interaction_id":iid,"decision":"once"}),
)
.await;
let next = replacement.pending(sid).await;
assert_eq!(next["turn"]["id"], second.turn_id);
assert_eq!(
api_a.submission(sid, "one").unwrap().status,
SubmissionStatus::Completed
);
replacement.rpc("interaction/respond",json!({"session_id":sid,"interaction_id":next["interactions"][0]["id"],"decision":"once"})).await;
tokio::time::timeout(Duration::from_secs(20), async {
while api_a.submission(sid, "two").unwrap().status != SubmissionStatus::Completed {
tokio::time::sleep(Duration::from_millis(50)).await;
}
})
.await
.unwrap();
let saved = api_b.inspect(sid).unwrap();
assert_eq!(
saved.history.iter().filter(|v| v["role"] == "user").count(),
2
);
assert_eq!(
std::fs::read_to_string(root.path().join("work/proof.txt")).unwrap(),
"one real side effect\n"
);
assert_eq!(
api_b.submit_turn(make("one", "write_once")).unwrap().status,
SubmissionStatus::Completed
);
// A completed duplicate never feeds another model turn or repeats the tool.
assert_eq!(
api_b.inspect(sid).unwrap().history.len(),
saved.history.len()
);
replacement.close().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[ignore = "requires isolated OW_PG_TEST_URL and native Linux"]
async fn real_worker_deaths_preserve_approval_then_deliver_unknown_without_replaying_write() {
let root = tempfile::tempdir().unwrap();
std::fs::create_dir_all(root.path().join("bundle/coworker")).unwrap();
std::fs::create_dir(root.path().join("work")).unwrap();
std::fs::write(
root.path().join("bundle/coworker/manifest.md"),
"---\nid: pg-test\nname: PG test\n---\nUse the test tool.",
)
.unwrap();
std::fs::write(
root.path().join("bundle/tools.json"),
r#"{"version":3,"tools":["test.probe"]}"#,
)
.unwrap();
let schema = format!("ow_worker_{}", uuid::Uuid::new_v4().simple());
let mut config = Config::new(std::env::var("OW_PG_TEST_URL").unwrap(), schema.clone());
config.tls = Tls::LoopbackPlaintext;
config.lease_duration = Duration::from_secs(6);
config.timeout = Duration::from_secs(1);
PostgresStore::migrate(&config).unwrap();
let mut a = Worker::start(root.path(), &schema, "worker-a", None).await;
let created = a
.rpc(
"session/create",
json!({"name":"Recovery","coworker":"pg-test","workspace":root.path().join("work")}),
)
.await;
let sid = created["session"]["id"].as_str().unwrap();
let plan = format!(
"TEST_RESOURCE_GRANTS {}",
json!([{"name":"test__probe","arguments":{"action":"write_wait"}}])
);
a.rpc(
"turn/start",
json!({"session_id":sid,"submission_id":"original","text":plan}),
)
.await;
let first = a.pending(sid).await;
let iid = first["interactions"][0]["id"].as_str().unwrap();
a.kill().await;
tokio::time::sleep(Duration::from_millis(6200)).await;
let mut b = Worker::start(root.path(), &schema, "worker-b", Some(sid)).await;
let pending = b.pending(sid).await;
assert_eq!(pending["interactions"][0]["id"], iid);
b.rpc(
"interaction/respond",
json!({"session_id":sid,"interaction_id":iid,"decision":"once"}),
)
.await;
tokio::time::timeout(Duration::from_secs(10), async {
while std::fs::read_to_string(root.path().join("work/proof.txt"))
.ok()
.as_deref()
!= Some("one real side effect\n")
{
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.unwrap();
b.kill().await;
tokio::time::sleep(Duration::from_millis(6200)).await;
let mut c = Worker::start(root.path(), &schema, "worker-c", Some(sid)).await;
let snap = c.rpc("session/attach", json!({"session_id":sid})).await;
assert_eq!(snap["turn"]["status"], "interrupted");
let observer = PostgresStore::open(config.clone()).unwrap();
let saved = observer.inspect(sid).unwrap();
assert_eq!(
saved
.history
.iter()
.filter(|v| v["type"] == "function_call_output"
&& v["output"]
.as_str()
.is_some_and(|s| s.contains("tool_outcome_unknown")))
.count(),
1
);
assert_eq!(
std::fs::read_to_string(root.path().join("work/proof.txt")).unwrap(),
"one real side effect\n"
);
// Budget reservations survive the same hard death as the uncertain write.
assert_eq!(saved.turn.as_ref().unwrap().budget.model_calls, 1);
assert!(saved.turn.as_ref().unwrap().budget.charged_output_tokens > 0);
assert!(saved.turn.as_ref().unwrap().queue.is_empty());
// Explicit continuation for this S1 test. Automatic scheduling/continuation is S3.
let inspection = format!(
"TEST_RESOURCE_GRANTS {}",
json!([{"name":"test__probe","arguments":{"action":"inspect"}}])
);
c.rpc(
"turn/start",
json!({"session_id":sid,"submission_id":"inspect-after-recovery","text":inspection}),
)
.await;
let pending = c.pending(sid).await;
c.rpc("interaction/respond",json!({"session_id":sid,"interaction_id":pending["interactions"][0]["id"],"decision":"once"})).await;
tokio::time::timeout(Duration::from_secs(10), async {
loop {
let v = c.rpc("session/attach", json!({"session_id":sid})).await;
if v["turn"]["status"] == "completed" {
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.unwrap();
assert!(observer
.inspect(sid)
.unwrap()
.history
.iter()
.any(|v| v["type"] == "function_call_output"
&& v["output"]
.as_str()
.is_some_and(|s| s.contains("observed") && s.contains("one real side effect"))));
assert_eq!(
std::fs::read_to_string(root.path().join("work/proof.txt")).unwrap(),
"one real side effect\n"
);
c.close().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[ignore = "requires isolated OW_PG_TEST_URL and native Linux"]
async fn resilience_streaming_worker_death_preserves_budget_and_cursor_without_tool_replay() {
let (root, config, api) = service_fixture();
std::fs::create_dir(root.path().join("work")).unwrap();
let mut seeder = Worker::start(root.path(), &config.schema, "seed", None).await;
let created=seeder.rpc("session/create",json!({"name":"Stream fault","coworker":"pg-test","workspace":root.path().join("work"),"permissions":{"mode":"bypass"}})).await;
let sid = created["session"]["id"].as_str().unwrap();
api.submit_turn(TurnSubmission {
session_id: sid.into(),
submission_id: "streaming".into(),
text: "TEST_RESILIENCE_STREAM".into(),
limits: Some(openworker_runtime::budget::Limits {
max_model_calls: 7,
..Default::default()
}),
})
.unwrap();
seeder.kill().await;
let mut a = service(root.path(), &config.schema, "stream-a", sid);
concurrency::wait(|| {
api.read_events(sid, 0)
.unwrap()
.iter()
.any(|e| e.kind == "assistant_delta")
})
.await;
let before = api.read_events(sid, 0).unwrap();
let cursor = before.last().unwrap().event_id;
let saved = api.inspect(sid).unwrap();
let budget = serde_json::to_value(&saved.turn.as_ref().unwrap().budget).unwrap();
let turn = saved.turn.as_ref().unwrap().id.clone();
a.kill().await.unwrap(); // SIGKILL an owned child, not a graceful drain.
a.wait().await.unwrap();
let mut b = service(root.path(), &config.schema, "stream-b", sid);
terminal_receipt(&api, sid, "streaming", SubmissionStatus::Interrupted).await;
let recovered = api.inspect(sid).unwrap();
assert_eq!(recovered.turn.as_ref().unwrap().id, turn);
assert_eq!(
serde_json::to_value(&recovered.turn.as_ref().unwrap().budget).unwrap(),
budget
);
assert_eq!(recovered.turn.as_ref().unwrap().budget.model_calls, 1);
assert_eq!(
recovered
.turn
.as_ref()
.unwrap()
.budget
.limits
.max_model_calls,
7
);
assert!(recovered.turn.as_ref().unwrap().queue.is_empty());
assert!(!recovered
.history
.iter()
.any(|v| v["type"] == "function_call"));
let tail = api.read_events(sid, cursor).unwrap();
assert!(!tail.iter().any(|e| e.kind == "tool_started"));
assert!(tail.iter().all(|e| e.event_id > cursor));
assert!(tail.windows(2).all(|w| w[0].event_id < w[1].event_id));
let count = api.read_events(sid, 0).unwrap().len();
// Reconnect/inspect is read-only: it cannot dispatch a second model request.
assert_eq!(api.read_snapshot(sid).unwrap()["turn"]["id"], turn);
assert_eq!(api.read_events(sid, 0).unwrap().len(), count);
stop_service(&mut b).await;
}