Skip to content
Python
23.5 KB
Raw Blame
use openworker_runtime::store::{Session, StateStore, Store};
use openworker_store_postgres::{Config, PostgresStore, Tls};
use serde_json::json;
use std::time::Duration;
fn config() -> Config {
    let url =
        std::env::var("OW_PG_TEST_URL").expect("Use a dedicated disposable PostgreSQL database");
    let mut c = Config::new(url, format!("ow_test_{}", uuid::Uuid::new_v4().simple()));
    c.tls = Tls::LoopbackPlaintext;
    c.timeout = Duration::from_secs(3);
    c
}
fn fixture() -> (Config, PostgresStore) {
    let c = config();
    PostgresStore::migrate(&c).unwrap();
    PostgresStore::migrate(&c).unwrap();
    let p = PostgresStore::open(c.clone()).unwrap();
    (c, p)
}
fn session(id: &str) -> Session {
    serde_json::from_value(json!({"id":id,"name":"Test","workspace":"/work","coworker":"test","bundle_digest":"bundle","schema_digest":null,"interactions":[],"turn":null,"sandbox":null,"closed":false,"resource_policy":openworker_runtime::resource_policy::Resolved::default()})).unwrap()
}
fn admin(c: &Config) -> postgres::Client {
    let mut db = postgres::Client::connect(&c.connection, postgres::NoTls).unwrap();
    db.batch_execute(&format!(
        "SET ow.runtime_schema='7'; SET search_path={},pg_catalog",
        c.schema
    ))
    .unwrap();
    db
}
fn expire(c: &Config, sid: &str) {
    admin(c)
        .execute(
            "UPDATE sessions SET expires=clock_timestamp()-interval '1 second' WHERE id=$1",
            &[&sid],
        )
        .unwrap();
}
fn exercise(store: &mut dyn StateStore) {
    store
        .save(&session("s_test"), "session_created", json!({}))
        .unwrap();
    let (s, fresh, _) = store.accept("s_test", "submission", "Hello").unwrap();
    assert!(fresh);
    let (again, fresh, _) = store.accept("s_test", "submission", "Hello").unwrap();
    assert!(!fresh);
    assert_eq!(s.turn.unwrap().id, again.turn.unwrap().id);
    assert!(store.accept("s_test", "submission", "Different").is_err());
    assert!(store.accept("s_test", "busy", "New task").is_err());
    store.start_attempt("s_test", "attempt", 1).unwrap();
    store.attempt_delta("s_test", "attempt", "abc").unwrap();
    store
        .finish_attempt("s_test", "attempt", "completed", None)
        .unwrap();
    assert_eq!(store.attempts("s_test").unwrap()[0]["text"], "abc");
    let mut s = store.get("s_test").unwrap();
    s.history
        .push(json!({"type":"reasoning","encrypted_content":"PRIVATE_REASONING"}));
    s.history.push(json!({"type":"message","role":"assistant","content":[{"type":"output_text","text":"Done"}]}));
    s.turn.as_mut().unwrap().status = "completed".into();
    store
        .save(&s, "turn_completed", json!({"status":"completed"}))
        .unwrap();
    let page = store.history_page("s_test", 0).unwrap();
    assert!(!page.to_string().contains("PRIVATE_REASONING"));
    assert_eq!(page["total_items"], 3);
    assert_eq!(page["items"].as_array().unwrap().len(), 2);
    assert_eq!(store.recent_messages("s_test").unwrap().0.len(), 2);
    assert!(!store.transcript_excerpt("s_test", 0, 0).unwrap()["text"]
        .as_str()
        .unwrap()
        .is_empty());
    let (new, fresh, _) = store.accept("s_test", "next", "Next task").unwrap();
    assert!(fresh);
    let (old, fresh, _) = store.accept("s_test", "submission", "Hello").unwrap();
    assert!(!fresh);
    assert_ne!(old.turn.unwrap().id, new.turn.unwrap().id);
    assert!(store.events("s_test", -1).is_err());
    assert!(store.events("s_test", i64::MAX).is_err());
    let events = store.events("s_test", 0).unwrap();
    assert_eq!(
        store.cursor("s_test").unwrap(),
        events.last().unwrap().event_id
    );
}

#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn imported_history_and_turn_context_survive_postgres_atomicity_and_takeover() {
    use openworker_runtime::history_import::{canonical, HistoryItem};
    use openworker_runtime::turn_options::{Context, OutputOptions};
    let (config, mut owner) = fixture();
    let mut s = session("s_import");
    s.auto_approver_context = Some("SESSION_REVIEW_DEFAULT".into());
    s.compaction_config.additional_instructions = Some("SESSION_SUMMARY_DEFAULT".into());
    s.history = canonical(&[
        HistoryItem::User {
            text: "Earlier request".into(),
        },
        HistoryItem::ToolCall {
            call_id: "old".into(),
            name: "app__lookup".into(),
            arguments: json!({}),
        },
        HistoryItem::ToolResult {
            call_id: "old".into(),
            output: "done".into(),
        },
    ])
    .unwrap();
    let mut sql = admin(&config);
    sql.batch_execute("CREATE FUNCTION reject_import_event() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'fixture'; END $$; CREATE TRIGGER reject_import_event BEFORE INSERT ON events FOR EACH ROW EXECUTE FUNCTION reject_import_event();").unwrap();
    assert!(owner.save(&s, "session_created", json!({})).is_err());
    assert_eq!(
        sql.query_one("SELECT count(*) FROM sessions", &[])
            .unwrap()
            .get::<_, i64>(0),
        0
    );
    assert_eq!(
        sql.query_one("SELECT count(*) FROM history", &[])
            .unwrap()
            .get::<_, i64>(0),
        0
    );
    sql.batch_execute(
        "DROP TRIGGER reject_import_event ON events; DROP FUNCTION reject_import_event();",
    )
    .unwrap();
    // A database fault deliberately fences this adapter. Reconcile rollback
    // through the independent admin connection above, then reconnect rather
    // than reusing a writer whose outcome was uncertain.
    assert!(owner.get(&s.id).is_err());
    drop(owner);
    let mut owner = PostgresStore::open(config.clone()).unwrap();
    owner.save(&s, "session_created", json!({})).unwrap();
    assert!(owner.get(&s.id).unwrap().turn.is_none());
    let context = Context {
        content: None,
        instruction_addendum: Some("TURN_PRIVATE_CONTEXT".into()),
        auto_approver_context: Some("TURN_REVIEW_GUIDANCE".into()),
        compaction: Some(
            openworker_runtime::compaction::Instructions::additional_instructions(
                "TURN_SUMMARY_GUIDANCE",
            ),
        ),
        output: OutputOptions {
            reasoning: true,
            buffer_attempts: true,
        },
    };
    owner
        .accept_with_context(&s.id, "first", "New request", None, None, context.clone())
        .unwrap();
    assert!(
        !owner
            .accept_with_context(&s.id, "first", "New request", None, None, context.clone())
            .unwrap()
            .1
    );
    assert!(owner.accept(&s.id, "first", "New request").is_err());
    owner.release(&s.id).unwrap();
    drop(owner);
    let mut replacement = PostgresStore::open(config).unwrap();
    replacement.claim(&s.id).unwrap();
    let mut restored = replacement.get(&s.id).unwrap();

    assert_eq!(restored.auto_approver_context, s.auto_approver_context);
    assert_eq!(
        restored.compaction_config.additional_instructions,
        s.compaction_config.additional_instructions
    );
    for marker in ["TURN_REVIEW_GUIDANCE", "TURN_SUMMARY_GUIDANCE"] {
        assert!(!serde_json::to_string(&restored.history)
            .unwrap()
            .contains(marker));
    }
    assert_eq!(restored.history[..3], s.history);
    assert_eq!(
        restored.turn.as_ref().unwrap().instruction_addendum,
        context.instruction_addendum
    );
    assert_eq!(restored.turn.as_ref().unwrap().output, context.output);
    assert_eq!(
        restored.turn.as_ref().unwrap().auto_approver_context,
        context.auto_approver_context
    );
    assert_eq!(
        restored.turn.as_ref().unwrap().compaction,
        context.compaction
    );
    for altered in [
        Context {
            auto_approver_context: Some("changed".into()),
            ..context.clone()
        },
        Context {
            compaction: Some(
                openworker_runtime::compaction::Instructions::additional_instructions("changed"),
            ),
            ..context.clone()
        },
    ] {
        assert!(replacement
            .accept_with_context(&s.id, "first", "New request", None, None, altered)
            .is_err());
    }

    assert!(!replacement
        .history_page(&s.id, 0)
        .unwrap()
        .to_string()
        .contains("TURN_PRIVATE_CONTEXT"));
    assert!(
        !serde_json::to_string(&replacement.events(&s.id, 0).unwrap())
            .unwrap()
            .contains("TURN_PRIVATE_CONTEXT")
    );
    restored.turn.as_mut().unwrap().status = "completed".into();
    replacement
        .save(&restored, "turn_completed", json!({"status":"completed"}))
        .unwrap();
    let (next, _, _) = replacement.accept(&s.id, "next", "Next request").unwrap();
    assert!(next.turn.as_ref().unwrap().instruction_addendum.is_none());
    assert!(next.turn.as_ref().unwrap().auto_approver_context.is_none());
    assert!(next.turn.as_ref().unwrap().compaction.is_none());
    assert_eq!(
        next.auto_approver_context.as_deref(),
        Some("SESSION_REVIEW_DEFAULT")
    );
    assert_eq!(
        next.compaction_config.additional_instructions.as_deref(),
        Some("SESSION_SUMMARY_DEFAULT")
    );
    assert_eq!(next.turn.as_ref().unwrap().output, OutputOptions::default());
    let (old, fresh, _) = replacement
        .accept_with_context(&s.id, "first", "New request", None, None, context.clone())
        .unwrap();
    assert!(!fresh);
    assert_eq!(
        old.turn.as_ref().unwrap().instruction_addendum,
        context.instruction_addendum
    );
    assert!(replacement
        .accept_with_context(
            &s.id,
            "first",
            "New request",
            None,
            None,
            Context {
                content: None,
                output: OutputOptions::default(),
                ..context
            }
        )
        .is_err());
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn sqlite_postgres_domain_conformance() {
    let tmp = tempfile::tempdir().unwrap();
    exercise(&mut Store::open(tmp.path()).unwrap());
    let (_, mut pg) = fixture();
    exercise(&mut pg);
}

#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn durable_plan_atomicity_non_owner_snapshot_and_fenced_takeover() {
    let (c, mut owner) = fixture();
    let api = PostgresStore::open(c.clone()).unwrap();
    let mut initial = session("s_plan");
    initial.package.planning = true;
    owner.save(&initial, "session_created", json!({})).unwrap();
    let (mut s, _, _) = owner.accept("s_plan", "submit", "Inspect only").unwrap();
    let tid = s.turn.as_ref().unwrap().id.clone();
    let args = json!({"expected_revision":0,"explanation":"PRIVATE_PLAN_MARKER","steps":[{"id":"inspect","description":"Read files","status":"in_progress"}]});
    let call = json!({"type":"function_call","name":"update_plan","call_id":"plan-call","arguments":args.to_string()});
    s.history.push(call.clone());
    s.turn.as_mut().unwrap().queue = vec![call];
    s.turn.as_mut().unwrap().status = "running".into();
    owner.save(&s, "fixture_queue", json!({})).unwrap();
    let mut s = owner.get("s_plan").unwrap();
    let before = s.clone();
    let plan = openworker_runtime::planning::next(None, &tid, "plan-call", &args).unwrap();
    s.plan = Some(plan.clone());
    s.history.push(json!({"type":"function_call_output","call_id":"plan-call","output":json!({"plan":plan,"advisory":true}).to_string()}));
    s.turn.as_mut().unwrap().queue.clear();
    let mut db = admin(&c);
    db.batch_execute("CREATE FUNCTION reject_plan_event() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'test failure'; END $$; CREATE TRIGGER reject_plan_event BEFORE INSERT ON events FOR EACH ROW EXECUTE FUNCTION reject_plan_event();").unwrap();
    assert!(owner
        .save(&s, "plan_updated", json!({"plan":plan}))
        .is_err());
    let failed = api.inspect("s_plan").unwrap();
    assert!(failed.plan.is_none());
    assert_eq!(failed.history, before.history);
    assert_eq!(
        failed.turn.as_ref().unwrap().queue,
        before.turn.as_ref().unwrap().queue
    );
    assert_eq!(failed.storage_revision, before.storage_revision);
    db.batch_execute("DROP TRIGGER reject_plan_event ON events")
        .unwrap();
    // A failed transaction poisons the adapter by design. Reconnect, fence the
    // failed owner and reconcile the retained queue before an explicit retry.
    assert!(owner.get("s_plan").is_err());
    expire(&c, "s_plan");
    owner = PostgresStore::open(c.clone()).unwrap();
    owner.claim("s_plan").unwrap();
    s.storage_revision = owner.get("s_plan").unwrap().storage_revision;
    let event = owner
        .save(&s, "plan_updated", json!({"plan":plan}))
        .unwrap();
    let snapshot = api.read_snapshot("s_plan").unwrap();
    assert_eq!(snapshot["plan"], json!(plan));
    assert_eq!(snapshot["cursor"], event.event_id);
    assert!(!api
        .session_page(None, 10)
        .unwrap()
        .to_string()
        .contains("PRIVATE_PLAN_MARKER"));
    let stale = owner.get("s_plan").unwrap();
    expire(&c, "s_plan");
    let mut next = PostgresStore::open(c).unwrap();
    next.claim("s_plan").unwrap();
    next.recover().unwrap();
    assert!(owner
        .save(&stale, "plan_updated", json!({"plan":plan}))
        .is_err());
    let recovered = next.get("s_plan").unwrap();
    assert_eq!(recovered.plan.as_ref(), Some(&plan));
    assert!(recovered.turn.as_ref().unwrap().queue.is_empty());
    assert_eq!(
        next.events("s_plan", 0)
            .unwrap()
            .iter()
            .filter(|e| e.kind == "plan_updated")
            .count(),
        1
    );
    assert_eq!(api.read_snapshot("s_plan").unwrap()["plan"], json!(plan));
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn revisions_ownership_and_recovery_are_session_scoped() {
    let (c, mut a) = fixture();
    let mut b = PostgresStore::open(c.clone()).unwrap();
    a.save(&session("s_a"), "session_created", json!({}))
        .unwrap();
    b.save(&session("s_b"), "session_created", json!({}))
        .unwrap();
    assert!(b.claim("s_a").is_err());
    assert!(b.get("s_a").is_err());
    let stale = a.get("s_a").unwrap();
    a.append_event("s_a", "progress", json!({})).unwrap();
    // Journal-only progress changes the event cursor, not canonical metadata.
    // A real metadata write must still invalidate the old snapshot.
    let mut updated = a.get("s_a").unwrap();
    assert_eq!(updated.storage_revision, stale.storage_revision);
    updated.name = "Renamed".into();
    a.save(&updated, "renamed", json!({})).unwrap();
    assert!(a.save(&stale, "stale", json!({})).is_err());
    a.accept("s_a", "a", "Hello").unwrap();
    b.accept("s_b", "b", "Other").unwrap();
    b.recover().unwrap();
    assert_eq!(a.get("s_a").unwrap().turn.unwrap().status, "accepted");
    let old = a.renew("s_a").unwrap();
    expire(&c, "s_a");
    let new = b.claim("s_a").unwrap();
    assert!(new.epoch > old.epoch);
    assert!(a.renew("s_a").is_err());
    assert!(a.append_event("s_a", "tool_started", json!({})).is_err());
    b.recover().unwrap();
    assert_eq!(b.get("s_a").unwrap().turn.unwrap().status, "interrupted");
    let cursor = b.cursor("s_a").unwrap();
    b.recover().unwrap();
    assert_eq!(b.cursor("s_a").unwrap(), cursor);
    assert_eq!(PostgresStore::open(c).unwrap().list().unwrap().len(), 0);
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn approval_and_unknown_result_survive_takeover_without_replay() {
    let (c, mut a) = fixture();
    a.save(&session("s_test"), "session_created", json!({}))
        .unwrap();
    let (mut s, _, _) = a.accept("s_test", "submit", "secret prompt").unwrap();
    let tid = s.turn.as_ref().unwrap().id.clone();
    let call = json!({"type":"function_call","call_id":"c_one","name":"write_file","arguments":"SECRET_ARGUMENTS"});
    s.history.push(call.clone());
    s.turn.as_mut().unwrap().queue = vec![call.clone()];
    s.turn.as_mut().unwrap().status = "waiting".into();
    s.interactions.push(serde_json::from_value(json!({"id":"i_one","turn_id":tid,"kind":"approval","call":call,"bundle_digest":"bundle","schema_digest":"schema","response":null,"resource_policy_digest":"resource"})).unwrap());
    a.save(
        &s,
        "interaction_required",
        json!({"private":"SECRET_ARGUMENTS"}),
    )
    .unwrap();
    a.release("s_test").unwrap();
    let mut b = PostgresStore::open(c.clone()).unwrap();
    b.claim("s_test").unwrap();
    b.recover().unwrap();
    let mut s = b.get("s_test").unwrap();
    assert_eq!(s.turn.as_ref().unwrap().status, "recoverable");
    assert_eq!(s.interactions[0].id, "i_one");
    assert!(s.interactions[0].response.is_none());
    s.interactions[0].response = Some("once".into());
    s.interactions[0].origin = Some("human".into());
    s.turn.as_mut().unwrap().status = "executing".into();
    b.save(
        &s,
        "tool_started",
        json!({"call_id":"c_one","arguments":"SECRET_ARGUMENTS"}),
    )
    .unwrap();
    expire(&c, "s_test");
    let mut next = PostgresStore::open(c.clone()).unwrap();
    next.claim("s_test").unwrap();
    next.recover().unwrap();
    let recovered = next.get("s_test").unwrap();
    assert_eq!(recovered.interactions[0].response.as_deref(), Some("once"));
    assert_eq!(recovered.turn.unwrap().status, "interrupted");
    let result = recovered.history.last().unwrap();
    assert_eq!(result["call_id"], "c_one");
    assert!(result["output"]
        .as_str()
        .unwrap()
        .contains("tool_outcome_unknown"));
    let length = recovered.history.len();
    next.recover().unwrap();
    assert_eq!(next.get("s_test").unwrap().history.len(), length);
    let audit = next.audit("s_test", 0).unwrap();
    let raw = serde_json::to_string(&audit).unwrap();
    assert!(!raw.contains("SECRET_ARGUMENTS"));
    assert!(!raw.contains("secret prompt"));
    assert_eq!(audit.len(), next.events("s_test", 0).unwrap().len());
    assert!(b
        .save(&s, "tool_finished", json!({"success":true}))
        .is_err());
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn audit_failure_rolls_back_state_history_and_event() {
    let (c, mut a) = fixture();
    a.save(&session("s_test"), "session_created", json!({}))
        .unwrap();
    let mut s = a.get("s_test").unwrap();
    s.name = "CHANGED".into();
    s.history
        .push(json!({"role":"user","content":"secret content"}));
    admin(&c).batch_execute("CREATE FUNCTION fail_audit() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'DO_NOT_EXPOSE_SQL_SECRET'; END $$; CREATE TRIGGER fail_audit BEFORE INSERT ON audit FOR EACH ROW EXECUTE FUNCTION fail_audit()").unwrap();
    let e = a
        .save(&s, "changed", json!({"private":"secret"}))
        .unwrap_err()
        .to_string();
    assert!(!e.contains("DO_NOT_EXPOSE_SQL_SECRET"));
    let b = PostgresStore::open(c.clone()).unwrap();
    let saved = b.inspect("s_test").unwrap();
    assert_eq!(saved.name, "Test");
    assert!(saved.history.is_empty());
    assert_eq!(saved.storage_revision, 1);
    assert_eq!(b.audit("s_test", 0).unwrap().len(), 1);
    assert!(a
        .get("s_test")
        .unwrap_err()
        .to_string()
        .contains("unusable"));
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn history_prefix_and_customer_deletion_fail_explicitly() {
    let (c, mut a) = fixture();
    a.save(&session("s_test"), "session_created", json!({}))
        .unwrap();
    a.accept("s_test", "s", "one").unwrap();
    let mut s = a.get("s_test").unwrap();
    s.history[0]["content"] = json!("rewritten");
    assert!(a.save(&s, "bad", json!({})).is_err());
    admin(&c)
        .batch_execute("DELETE FROM audit WHERE cursor=1; DELETE FROM events WHERE cursor=1")
        .unwrap();
    assert!(a
        .events("s_test", 0)
        .unwrap_err()
        .to_string()
        .contains("gap"));
    assert!(a
        .audit("s_test", 0)
        .unwrap_err()
        .to_string()
        .contains("gap"));
    assert!(a.audit("s_test", -1).is_err());
    admin(&c)
        .execute("DELETE FROM history WHERE session_id='s_test'", &[])
        .unwrap();
    assert!(a.get("s_test").unwrap_err().to_string().contains("missing"));
    assert!(a.history_page("s_test", 0).is_err());
}

#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn blocked_database_operation_is_bounded_and_never_retried() {
    let (mut c, mut a) = fixture();
    a.save(&session("s_test"), "session_created", json!({}))
        .unwrap();
    a.release("s_test").unwrap();
    c.timeout = std::time::Duration::from_millis(800);
    let mut worker = PostgresStore::open(c.clone()).unwrap();
    worker.claim("s_test").unwrap();
    let before = worker.cursor("s_test").unwrap();
    let mut client = admin(&c);
    let mut blocker = client.transaction().unwrap();
    blocker
        .query_one("SELECT id FROM sessions WHERE id='s_test' FOR UPDATE", &[])
        .unwrap();
    let start = std::time::Instant::now();
    assert!(worker
        .append_event("s_test", "must_not_commit", json!({}))
        .is_err());
    assert!(start.elapsed() < std::time::Duration::from_secs(3));
    blocker.rollback().unwrap();
    assert!(worker.get("s_test").is_err());
    let observer = PostgresStore::open(c).unwrap();
    assert_eq!(observer.inspect("s_test").unwrap().storage_revision, before);
    assert_eq!(observer.audit("s_test", before).unwrap().len(), 0);
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn concurrent_claims_have_exactly_one_owner() {
    let (c, mut a) = fixture();
    a.save(&session("s_test"), "session_created", json!({}))
        .unwrap();
    a.release("s_test").unwrap();
    let b = PostgresStore::open(c.clone()).unwrap();
    let d = PostgresStore::open(c).unwrap();
    let barrier = std::sync::Arc::new(std::sync::Barrier::new(3));
    let b1 = barrier.clone();
    let b2 = barrier.clone();
    let t1 = std::thread::spawn(move || {
        b1.wait();
        b.claim("s_test").is_ok()
    });
    let t2 = std::thread::spawn(move || {
        b2.wait();
        d.claim("s_test").is_ok()
    });
    barrier.wait();
    assert_eq!(
        u8::from(t1.join().unwrap()) + u8::from(t2.join().unwrap()),
        1
    );
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn schema_tls_and_executor_modes_fail_closed() {
    let (c, a) = fixture();
    let mut bad = c.clone();
    bad.schema = "public;DROP".into();
    assert!(PostgresStore::open(bad).is_err());
    let mut secure = c.clone();
    secure.tls = Tls::Verified { ca_file: None };
    assert!(PostgresStore::open(secure).is_err());
    let mut remote = c.clone();
    remote.connection = zeroize::Zeroizing::new("host=example.com user=postgres".into());
    assert!(PostgresStore::open(remote).is_err());
    let rt = tokio::runtime::Builder::new_current_thread()
        .enable_all()
        .build()
        .unwrap();
    rt.block_on(async {
        assert!(a.list().unwrap_err().to_string().contains("multi-threaded"));
    });
    admin(&c)
        .execute("UPDATE version SET version=99", &[])
        .unwrap();
    assert!(PostgresStore::migrate(&c).is_err());
    assert!(PostgresStore::open(c).is_err());
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn lost_database_connection_is_poisoned_without_retry() {
    let (c, mut a) = fixture();
    a.save(&session("s_test"), "session_created", json!({}))
        .unwrap();
    let lease = a.renew("s_test").unwrap();
    admin(&c)
        .query(
            "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name=$1",
            &[&format!("ow-store-{}", lease.owner)],
        )
        .unwrap();
    assert!(a.append_event("s_test", "tool_started", json!({})).is_err());
    assert!(a.get("s_test").is_err());
    let fresh = PostgresStore::open(c).unwrap();
    assert_eq!(fresh.inspect("s_test").unwrap().storage_revision, 1);
}