Python
use openworker_runtime::store::{Session, StateStore, Store};
use openworker_store_postgres::{Config, PostgresStore, Tls};
use serde_json::json;
use std::time::Duration;
fn config() -> Config {
let url =
std::env::var("OW_PG_TEST_URL").expect("Use a dedicated disposable PostgreSQL database");
let mut c = Config::new(url, format!("ow_test_{}", uuid::Uuid::new_v4().simple()));
c.tls = Tls::LoopbackPlaintext;
c.timeout = Duration::from_secs(3);
c
}
fn fixture() -> (Config, PostgresStore) {
let c = config();
PostgresStore::migrate(&c).unwrap();
PostgresStore::migrate(&c).unwrap();
let p = PostgresStore::open(c.clone()).unwrap();
(c, p)
}
fn session(id: &str) -> Session {
serde_json::from_value(json!({"id":id,"name":"Test","workspace":"/work","coworker":"test","bundle_digest":"bundle","schema_digest":null,"interactions":[],"turn":null,"sandbox":null,"closed":false,"resource_policy":openworker_runtime::resource_policy::Resolved::default()})).unwrap()
}
fn admin(c: &Config) -> postgres::Client {
let mut db = postgres::Client::connect(&c.connection, postgres::NoTls).unwrap();
db.batch_execute(&format!(
"SET ow.runtime_schema='7'; SET search_path={},pg_catalog",
c.schema
))
.unwrap();
db
}
fn expire(c: &Config, sid: &str) {
admin(c)
.execute(
"UPDATE sessions SET expires=clock_timestamp()-interval '1 second' WHERE id=$1",
&[&sid],
)
.unwrap();
}
fn exercise(store: &mut dyn StateStore) {
store
.save(&session("s_test"), "session_created", json!({}))
.unwrap();
let (s, fresh, _) = store.accept("s_test", "submission", "Hello").unwrap();
assert!(fresh);
let (again, fresh, _) = store.accept("s_test", "submission", "Hello").unwrap();
assert!(!fresh);
assert_eq!(s.turn.unwrap().id, again.turn.unwrap().id);
assert!(store.accept("s_test", "submission", "Different").is_err());
assert!(store.accept("s_test", "busy", "New task").is_err());
store.start_attempt("s_test", "attempt", 1).unwrap();
store.attempt_delta("s_test", "attempt", "abc").unwrap();
store
.finish_attempt("s_test", "attempt", "completed", None)
.unwrap();
assert_eq!(store.attempts("s_test").unwrap()[0]["text"], "abc");
let mut s = store.get("s_test").unwrap();
s.history
.push(json!({"type":"reasoning","encrypted_content":"PRIVATE_REASONING"}));
s.history.push(json!({"type":"message","role":"assistant","content":[{"type":"output_text","text":"Done"}]}));
s.turn.as_mut().unwrap().status = "completed".into();
store
.save(&s, "turn_completed", json!({"status":"completed"}))
.unwrap();
let page = store.history_page("s_test", 0).unwrap();
assert!(!page.to_string().contains("PRIVATE_REASONING"));
assert_eq!(page["total_items"], 3);
assert_eq!(page["items"].as_array().unwrap().len(), 2);
assert_eq!(store.recent_messages("s_test").unwrap().0.len(), 2);
assert!(!store.transcript_excerpt("s_test", 0, 0).unwrap()["text"]
.as_str()
.unwrap()
.is_empty());
let (new, fresh, _) = store.accept("s_test", "next", "Next task").unwrap();
assert!(fresh);
let (old, fresh, _) = store.accept("s_test", "submission", "Hello").unwrap();
assert!(!fresh);
assert_ne!(old.turn.unwrap().id, new.turn.unwrap().id);
assert!(store.events("s_test", -1).is_err());
assert!(store.events("s_test", i64::MAX).is_err());
let events = store.events("s_test", 0).unwrap();
assert_eq!(
store.cursor("s_test").unwrap(),
events.last().unwrap().event_id
);
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn imported_history_and_turn_context_survive_postgres_atomicity_and_takeover() {
use openworker_runtime::history_import::{canonical, HistoryItem};
use openworker_runtime::turn_options::{Context, OutputOptions};
let (config, mut owner) = fixture();
let mut s = session("s_import");
s.auto_approver_context = Some("SESSION_REVIEW_DEFAULT".into());
s.compaction_config.additional_instructions = Some("SESSION_SUMMARY_DEFAULT".into());
s.history = canonical(&[
HistoryItem::User {
text: "Earlier request".into(),
},
HistoryItem::ToolCall {
call_id: "old".into(),
name: "app__lookup".into(),
arguments: json!({}),
},
HistoryItem::ToolResult {
call_id: "old".into(),
output: "done".into(),
},
])
.unwrap();
let mut sql = admin(&config);
sql.batch_execute("CREATE FUNCTION reject_import_event() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'fixture'; END $$; CREATE TRIGGER reject_import_event BEFORE INSERT ON events FOR EACH ROW EXECUTE FUNCTION reject_import_event();").unwrap();
assert!(owner.save(&s, "session_created", json!({})).is_err());
assert_eq!(
sql.query_one("SELECT count(*) FROM sessions", &[])
.unwrap()
.get::<_, i64>(0),
0
);
assert_eq!(
sql.query_one("SELECT count(*) FROM history", &[])
.unwrap()
.get::<_, i64>(0),
0
);
sql.batch_execute(
"DROP TRIGGER reject_import_event ON events; DROP FUNCTION reject_import_event();",
)
.unwrap();
// A database fault deliberately fences this adapter. Reconcile rollback
// through the independent admin connection above, then reconnect rather
// than reusing a writer whose outcome was uncertain.
assert!(owner.get(&s.id).is_err());
drop(owner);
let mut owner = PostgresStore::open(config.clone()).unwrap();
owner.save(&s, "session_created", json!({})).unwrap();
assert!(owner.get(&s.id).unwrap().turn.is_none());
let context = Context {
content: None,
instruction_addendum: Some("TURN_PRIVATE_CONTEXT".into()),
auto_approver_context: Some("TURN_REVIEW_GUIDANCE".into()),
compaction: Some(
openworker_runtime::compaction::Instructions::additional_instructions(
"TURN_SUMMARY_GUIDANCE",
),
),
output: OutputOptions {
reasoning: true,
buffer_attempts: true,
},
};
owner
.accept_with_context(&s.id, "first", "New request", None, None, context.clone())
.unwrap();
assert!(
!owner
.accept_with_context(&s.id, "first", "New request", None, None, context.clone())
.unwrap()
.1
);
assert!(owner.accept(&s.id, "first", "New request").is_err());
owner.release(&s.id).unwrap();
drop(owner);
let mut replacement = PostgresStore::open(config).unwrap();
replacement.claim(&s.id).unwrap();
let mut restored = replacement.get(&s.id).unwrap();
assert_eq!(restored.auto_approver_context, s.auto_approver_context);
assert_eq!(
restored.compaction_config.additional_instructions,
s.compaction_config.additional_instructions
);
for marker in ["TURN_REVIEW_GUIDANCE", "TURN_SUMMARY_GUIDANCE"] {
assert!(!serde_json::to_string(&restored.history)
.unwrap()
.contains(marker));
}
assert_eq!(restored.history[..3], s.history);
assert_eq!(
restored.turn.as_ref().unwrap().instruction_addendum,
context.instruction_addendum
);
assert_eq!(restored.turn.as_ref().unwrap().output, context.output);
assert_eq!(
restored.turn.as_ref().unwrap().auto_approver_context,
context.auto_approver_context
);
assert_eq!(
restored.turn.as_ref().unwrap().compaction,
context.compaction
);
for altered in [
Context {
auto_approver_context: Some("changed".into()),
..context.clone()
},
Context {
compaction: Some(
openworker_runtime::compaction::Instructions::additional_instructions("changed"),
),
..context.clone()
},
] {
assert!(replacement
.accept_with_context(&s.id, "first", "New request", None, None, altered)
.is_err());
}
assert!(!replacement
.history_page(&s.id, 0)
.unwrap()
.to_string()
.contains("TURN_PRIVATE_CONTEXT"));
assert!(
!serde_json::to_string(&replacement.events(&s.id, 0).unwrap())
.unwrap()
.contains("TURN_PRIVATE_CONTEXT")
);
restored.turn.as_mut().unwrap().status = "completed".into();
replacement
.save(&restored, "turn_completed", json!({"status":"completed"}))
.unwrap();
let (next, _, _) = replacement.accept(&s.id, "next", "Next request").unwrap();
assert!(next.turn.as_ref().unwrap().instruction_addendum.is_none());
assert!(next.turn.as_ref().unwrap().auto_approver_context.is_none());
assert!(next.turn.as_ref().unwrap().compaction.is_none());
assert_eq!(
next.auto_approver_context.as_deref(),
Some("SESSION_REVIEW_DEFAULT")
);
assert_eq!(
next.compaction_config.additional_instructions.as_deref(),
Some("SESSION_SUMMARY_DEFAULT")
);
assert_eq!(next.turn.as_ref().unwrap().output, OutputOptions::default());
let (old, fresh, _) = replacement
.accept_with_context(&s.id, "first", "New request", None, None, context.clone())
.unwrap();
assert!(!fresh);
assert_eq!(
old.turn.as_ref().unwrap().instruction_addendum,
context.instruction_addendum
);
assert!(replacement
.accept_with_context(
&s.id,
"first",
"New request",
None,
None,
Context {
content: None,
output: OutputOptions::default(),
..context
}
)
.is_err());
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn sqlite_postgres_domain_conformance() {
let tmp = tempfile::tempdir().unwrap();
exercise(&mut Store::open(tmp.path()).unwrap());
let (_, mut pg) = fixture();
exercise(&mut pg);
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn durable_plan_atomicity_non_owner_snapshot_and_fenced_takeover() {
let (c, mut owner) = fixture();
let api = PostgresStore::open(c.clone()).unwrap();
let mut initial = session("s_plan");
initial.package.planning = true;
owner.save(&initial, "session_created", json!({})).unwrap();
let (mut s, _, _) = owner.accept("s_plan", "submit", "Inspect only").unwrap();
let tid = s.turn.as_ref().unwrap().id.clone();
let args = json!({"expected_revision":0,"explanation":"PRIVATE_PLAN_MARKER","steps":[{"id":"inspect","description":"Read files","status":"in_progress"}]});
let call = json!({"type":"function_call","name":"update_plan","call_id":"plan-call","arguments":args.to_string()});
s.history.push(call.clone());
s.turn.as_mut().unwrap().queue = vec![call];
s.turn.as_mut().unwrap().status = "running".into();
owner.save(&s, "fixture_queue", json!({})).unwrap();
let mut s = owner.get("s_plan").unwrap();
let before = s.clone();
let plan = openworker_runtime::planning::next(None, &tid, "plan-call", &args).unwrap();
s.plan = Some(plan.clone());
s.history.push(json!({"type":"function_call_output","call_id":"plan-call","output":json!({"plan":plan,"advisory":true}).to_string()}));
s.turn.as_mut().unwrap().queue.clear();
let mut db = admin(&c);
db.batch_execute("CREATE FUNCTION reject_plan_event() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'test failure'; END $$; CREATE TRIGGER reject_plan_event BEFORE INSERT ON events FOR EACH ROW EXECUTE FUNCTION reject_plan_event();").unwrap();
assert!(owner
.save(&s, "plan_updated", json!({"plan":plan}))
.is_err());
let failed = api.inspect("s_plan").unwrap();
assert!(failed.plan.is_none());
assert_eq!(failed.history, before.history);
assert_eq!(
failed.turn.as_ref().unwrap().queue,
before.turn.as_ref().unwrap().queue
);
assert_eq!(failed.storage_revision, before.storage_revision);
db.batch_execute("DROP TRIGGER reject_plan_event ON events")
.unwrap();
// A failed transaction poisons the adapter by design. Reconnect, fence the
// failed owner and reconcile the retained queue before an explicit retry.
assert!(owner.get("s_plan").is_err());
expire(&c, "s_plan");
owner = PostgresStore::open(c.clone()).unwrap();
owner.claim("s_plan").unwrap();
s.storage_revision = owner.get("s_plan").unwrap().storage_revision;
let event = owner
.save(&s, "plan_updated", json!({"plan":plan}))
.unwrap();
let snapshot = api.read_snapshot("s_plan").unwrap();
assert_eq!(snapshot["plan"], json!(plan));
assert_eq!(snapshot["cursor"], event.event_id);
assert!(!api
.session_page(None, 10)
.unwrap()
.to_string()
.contains("PRIVATE_PLAN_MARKER"));
let stale = owner.get("s_plan").unwrap();
expire(&c, "s_plan");
let mut next = PostgresStore::open(c).unwrap();
next.claim("s_plan").unwrap();
next.recover().unwrap();
assert!(owner
.save(&stale, "plan_updated", json!({"plan":plan}))
.is_err());
let recovered = next.get("s_plan").unwrap();
assert_eq!(recovered.plan.as_ref(), Some(&plan));
assert!(recovered.turn.as_ref().unwrap().queue.is_empty());
assert_eq!(
next.events("s_plan", 0)
.unwrap()
.iter()
.filter(|e| e.kind == "plan_updated")
.count(),
1
);
assert_eq!(api.read_snapshot("s_plan").unwrap()["plan"], json!(plan));
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn revisions_ownership_and_recovery_are_session_scoped() {
let (c, mut a) = fixture();
let mut b = PostgresStore::open(c.clone()).unwrap();
a.save(&session("s_a"), "session_created", json!({}))
.unwrap();
b.save(&session("s_b"), "session_created", json!({}))
.unwrap();
assert!(b.claim("s_a").is_err());
assert!(b.get("s_a").is_err());
let stale = a.get("s_a").unwrap();
a.append_event("s_a", "progress", json!({})).unwrap();
// Journal-only progress changes the event cursor, not canonical metadata.
// A real metadata write must still invalidate the old snapshot.
let mut updated = a.get("s_a").unwrap();
assert_eq!(updated.storage_revision, stale.storage_revision);
updated.name = "Renamed".into();
a.save(&updated, "renamed", json!({})).unwrap();
assert!(a.save(&stale, "stale", json!({})).is_err());
a.accept("s_a", "a", "Hello").unwrap();
b.accept("s_b", "b", "Other").unwrap();
b.recover().unwrap();
assert_eq!(a.get("s_a").unwrap().turn.unwrap().status, "accepted");
let old = a.renew("s_a").unwrap();
expire(&c, "s_a");
let new = b.claim("s_a").unwrap();
assert!(new.epoch > old.epoch);
assert!(a.renew("s_a").is_err());
assert!(a.append_event("s_a", "tool_started", json!({})).is_err());
b.recover().unwrap();
assert_eq!(b.get("s_a").unwrap().turn.unwrap().status, "interrupted");
let cursor = b.cursor("s_a").unwrap();
b.recover().unwrap();
assert_eq!(b.cursor("s_a").unwrap(), cursor);
assert_eq!(PostgresStore::open(c).unwrap().list().unwrap().len(), 0);
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn approval_and_unknown_result_survive_takeover_without_replay() {
let (c, mut a) = fixture();
a.save(&session("s_test"), "session_created", json!({}))
.unwrap();
let (mut s, _, _) = a.accept("s_test", "submit", "secret prompt").unwrap();
let tid = s.turn.as_ref().unwrap().id.clone();
let call = json!({"type":"function_call","call_id":"c_one","name":"write_file","arguments":"SECRET_ARGUMENTS"});
s.history.push(call.clone());
s.turn.as_mut().unwrap().queue = vec![call.clone()];
s.turn.as_mut().unwrap().status = "waiting".into();
s.interactions.push(serde_json::from_value(json!({"id":"i_one","turn_id":tid,"kind":"approval","call":call,"bundle_digest":"bundle","schema_digest":"schema","response":null,"resource_policy_digest":"resource"})).unwrap());
a.save(
&s,
"interaction_required",
json!({"private":"SECRET_ARGUMENTS"}),
)
.unwrap();
a.release("s_test").unwrap();
let mut b = PostgresStore::open(c.clone()).unwrap();
b.claim("s_test").unwrap();
b.recover().unwrap();
let mut s = b.get("s_test").unwrap();
assert_eq!(s.turn.as_ref().unwrap().status, "recoverable");
assert_eq!(s.interactions[0].id, "i_one");
assert!(s.interactions[0].response.is_none());
s.interactions[0].response = Some("once".into());
s.interactions[0].origin = Some("human".into());
s.turn.as_mut().unwrap().status = "executing".into();
b.save(
&s,
"tool_started",
json!({"call_id":"c_one","arguments":"SECRET_ARGUMENTS"}),
)
.unwrap();
expire(&c, "s_test");
let mut next = PostgresStore::open(c.clone()).unwrap();
next.claim("s_test").unwrap();
next.recover().unwrap();
let recovered = next.get("s_test").unwrap();
assert_eq!(recovered.interactions[0].response.as_deref(), Some("once"));
assert_eq!(recovered.turn.unwrap().status, "interrupted");
let result = recovered.history.last().unwrap();
assert_eq!(result["call_id"], "c_one");
assert!(result["output"]
.as_str()
.unwrap()
.contains("tool_outcome_unknown"));
let length = recovered.history.len();
next.recover().unwrap();
assert_eq!(next.get("s_test").unwrap().history.len(), length);
let audit = next.audit("s_test", 0).unwrap();
let raw = serde_json::to_string(&audit).unwrap();
assert!(!raw.contains("SECRET_ARGUMENTS"));
assert!(!raw.contains("secret prompt"));
assert_eq!(audit.len(), next.events("s_test", 0).unwrap().len());
assert!(b
.save(&s, "tool_finished", json!({"success":true}))
.is_err());
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn audit_failure_rolls_back_state_history_and_event() {
let (c, mut a) = fixture();
a.save(&session("s_test"), "session_created", json!({}))
.unwrap();
let mut s = a.get("s_test").unwrap();
s.name = "CHANGED".into();
s.history
.push(json!({"role":"user","content":"secret content"}));
admin(&c).batch_execute("CREATE FUNCTION fail_audit() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'DO_NOT_EXPOSE_SQL_SECRET'; END $$; CREATE TRIGGER fail_audit BEFORE INSERT ON audit FOR EACH ROW EXECUTE FUNCTION fail_audit()").unwrap();
let e = a
.save(&s, "changed", json!({"private":"secret"}))
.unwrap_err()
.to_string();
assert!(!e.contains("DO_NOT_EXPOSE_SQL_SECRET"));
let b = PostgresStore::open(c.clone()).unwrap();
let saved = b.inspect("s_test").unwrap();
assert_eq!(saved.name, "Test");
assert!(saved.history.is_empty());
assert_eq!(saved.storage_revision, 1);
assert_eq!(b.audit("s_test", 0).unwrap().len(), 1);
assert!(a
.get("s_test")
.unwrap_err()
.to_string()
.contains("unusable"));
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn history_prefix_and_customer_deletion_fail_explicitly() {
let (c, mut a) = fixture();
a.save(&session("s_test"), "session_created", json!({}))
.unwrap();
a.accept("s_test", "s", "one").unwrap();
let mut s = a.get("s_test").unwrap();
s.history[0]["content"] = json!("rewritten");
assert!(a.save(&s, "bad", json!({})).is_err());
admin(&c)
.batch_execute("DELETE FROM audit WHERE cursor=1; DELETE FROM events WHERE cursor=1")
.unwrap();
assert!(a
.events("s_test", 0)
.unwrap_err()
.to_string()
.contains("gap"));
assert!(a
.audit("s_test", 0)
.unwrap_err()
.to_string()
.contains("gap"));
assert!(a.audit("s_test", -1).is_err());
admin(&c)
.execute("DELETE FROM history WHERE session_id='s_test'", &[])
.unwrap();
assert!(a.get("s_test").unwrap_err().to_string().contains("missing"));
assert!(a.history_page("s_test", 0).is_err());
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn blocked_database_operation_is_bounded_and_never_retried() {
let (mut c, mut a) = fixture();
a.save(&session("s_test"), "session_created", json!({}))
.unwrap();
a.release("s_test").unwrap();
c.timeout = std::time::Duration::from_millis(800);
let mut worker = PostgresStore::open(c.clone()).unwrap();
worker.claim("s_test").unwrap();
let before = worker.cursor("s_test").unwrap();
let mut client = admin(&c);
let mut blocker = client.transaction().unwrap();
blocker
.query_one("SELECT id FROM sessions WHERE id='s_test' FOR UPDATE", &[])
.unwrap();
let start = std::time::Instant::now();
assert!(worker
.append_event("s_test", "must_not_commit", json!({}))
.is_err());
assert!(start.elapsed() < std::time::Duration::from_secs(3));
blocker.rollback().unwrap();
assert!(worker.get("s_test").is_err());
let observer = PostgresStore::open(c).unwrap();
assert_eq!(observer.inspect("s_test").unwrap().storage_revision, before);
assert_eq!(observer.audit("s_test", before).unwrap().len(), 0);
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn concurrent_claims_have_exactly_one_owner() {
let (c, mut a) = fixture();
a.save(&session("s_test"), "session_created", json!({}))
.unwrap();
a.release("s_test").unwrap();
let b = PostgresStore::open(c.clone()).unwrap();
let d = PostgresStore::open(c).unwrap();
let barrier = std::sync::Arc::new(std::sync::Barrier::new(3));
let b1 = barrier.clone();
let b2 = barrier.clone();
let t1 = std::thread::spawn(move || {
b1.wait();
b.claim("s_test").is_ok()
});
let t2 = std::thread::spawn(move || {
b2.wait();
d.claim("s_test").is_ok()
});
barrier.wait();
assert_eq!(
u8::from(t1.join().unwrap()) + u8::from(t2.join().unwrap()),
1
);
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn schema_tls_and_executor_modes_fail_closed() {
let (c, a) = fixture();
let mut bad = c.clone();
bad.schema = "public;DROP".into();
assert!(PostgresStore::open(bad).is_err());
let mut secure = c.clone();
secure.tls = Tls::Verified { ca_file: None };
assert!(PostgresStore::open(secure).is_err());
let mut remote = c.clone();
remote.connection = zeroize::Zeroizing::new("host=example.com user=postgres".into());
assert!(PostgresStore::open(remote).is_err());
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
rt.block_on(async {
assert!(a.list().unwrap_err().to_string().contains("multi-threaded"));
});
admin(&c)
.execute("UPDATE version SET version=99", &[])
.unwrap();
assert!(PostgresStore::migrate(&c).is_err());
assert!(PostgresStore::open(c).is_err());
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn lost_database_connection_is_poisoned_without_retry() {
let (c, mut a) = fixture();
a.save(&session("s_test"), "session_created", json!({}))
.unwrap();
let lease = a.renew("s_test").unwrap();
admin(&c)
.query(
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name=$1",
&[&format!("ow-store-{}", lease.owner)],
)
.unwrap();
assert!(a.append_event("s_test", "tool_started", json!({})).is_err());
assert!(a.get("s_test").is_err());
let fresh = PostgresStore::open(c).unwrap();
assert_eq!(fresh.inspect("s_test").unwrap().storage_revision, 1);
}