Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

36.7 KB
Raw Blame
use super::*;
use crate::core::FileSystem;
use crate::lock::{PathLockConfig, PathLockManager};
use crate::plugins::memfs::MemFileSystem;
use std::process::{Child, Command};
use std::time::Instant;
use tempfile::TempDir;

fn provider(dir: &TempDir, namespace: &str) -> SQLitePathLockProvider {
    SQLitePathLockProvider::new(dir.path().join("locks.db").to_str().unwrap(), namespace).unwrap()
}

fn request(path: &str, kind: PathLockKind) -> PathLockRequest {
    PathLockRequest {
        path: path.into(),
        kind,
    }
}

async fn acquire(
    provider: &SQLitePathLockProvider,
    path: &str,
    kind: PathLockKind,
    owner: &str,
    now: u128,
    stale: u128,
) -> PathLockResult<Vec<AtomicAcquisition>> {
    provider
        .try_acquire_batch_atomic(&[request(path, kind)], owner, now, stale)
        .await
        .map(Option::unwrap)
}

#[tokio::test]
async fn exact_and_tree_conflicts_are_shared_between_connections() {
    let dir = TempDir::new().unwrap();
    let first = provider(&dir, "test");
    let second = provider(&dir, "test");
    acquire(&first, "/a", PathLockKind::Exact, "one", 100, 0)
        .await
        .unwrap();
    // An ancestor Exact does not cover its children.
    acquire(&second, "/a/b", PathLockKind::Exact, "two", 100, 0)
        .await
        .unwrap();
    assert!(matches!(
        acquire(&second, "/a", PathLockKind::Tree, "two", 100, 0).await,
        Err(PathLockError::Conflict { .. })
    ));
    assert!(
        matches!(acquire(&first, "/a", PathLockKind::Tree, "one", 100, 0).await,
        Err(PathLockError::Conflict { lock_path, .. }) if lock_path == "/a/b")
    );
    second.remove_token("/a/b", "two", false).await.unwrap();
    acquire(&first, "/a", PathLockKind::Tree, "one", 101, 0)
        .await
        .unwrap();
    assert!(matches!(
        acquire(&second, "/a/b", PathLockKind::Exact, "two", 101, 0).await,
        Err(PathLockError::Conflict {
            kind: PathLockKind::Tree,
            ..
        })
    ));
    assert_eq!(
        second
            .is_path_locked_atomic("/a/b", 101, 0, true)
            .await
            .unwrap(),
        Some(true)
    );
}

#[tokio::test]
async fn prefix_boundaries_wildcards_unicode_root_and_namespaces() {
    let dir = TempDir::new().unwrap();
    let locks = provider(&dir, "one");
    for path in ["/ab/x", "/a%/x", "/a_/x", "/树/x", "/local/account2/x"] {
        acquire(&locks, path, PathLockKind::Exact, "one", 100, 0)
            .await
            .unwrap();
    }
    for path in ["/a", "/a%", "/a_", "/树", "/local/account2"] {
        let result = acquire(&locks, path, PathLockKind::Tree, "two", 100, 0).await;
        if path == "/a" {
            result.unwrap();
        } else {
            assert!(matches!(result, Err(PathLockError::Conflict { .. })));
        }
    }
    assert!(locks.scan_descendant_locks("/a").await.unwrap().is_empty());
    assert_eq!(locks.scan_descendant_locks("/").await.unwrap().len(), 6);
    assert!(matches!(
        acquire(&locks, "/", PathLockKind::Tree, "three", 100, 0).await,
        Err(PathLockError::Conflict { .. })
    ));
    // Separate services in the same DB do not conflict; account paths remain distinct.
    acquire(
        &provider(&dir, "two"),
        "/",
        PathLockKind::Tree,
        "three",
        100,
        0,
    )
    .await
    .unwrap();
    acquire(
        &locks,
        "/local/account1",
        PathLockKind::Tree,
        "three",
        100,
        0,
    )
    .await
    .unwrap();
}

#[tokio::test]
async fn stale_takeover_preserves_u128_boundary_and_ownership() {
    let dir = TempDir::new().unwrap();
    let locks = provider(&dir, "test");
    acquire(&locks, "/a/b", PathLockKind::Exact, "old", u128::MAX - 1, 0)
        .await
        .unwrap();
    assert!(matches!(
        acquire(
            &locks,
            "/a",
            PathLockKind::Tree,
            "new",
            u128::MAX,
            u128::MAX - 1
        )
        .await,
        Err(PathLockError::Conflict { .. })
    ));
    acquire(
        &locks,
        "/a",
        PathLockKind::Tree,
        "new",
        u128::MAX,
        u128::MAX,
    )
    .await
    .unwrap();
    assert_eq!(
        locks.read_token("/a").await.unwrap().unwrap().time_ns,
        u128::MAX
    );
    acquire(
        &locks,
        "/a/b",
        PathLockKind::Exact,
        "new",
        u128::MAX,
        u128::MAX,
    )
    .await
    .unwrap();
    assert!(!locks.refresh_token("/a/b", "old", 0).await.unwrap());
    assert!(!locks.remove_token("/a/b", "old", true).await.unwrap());
    assert!(locks.refresh_token("/a/b", "new", 9).await.unwrap());
    assert_eq!(
        locks
            .is_path_locked_atomic("/a", u128::MAX, u128::MAX, true)
            .await
            .unwrap(),
        Some(true)
    );
    locks.remove_token("/a", "new", false).await.unwrap();
    assert_eq!(
        locks
            .is_path_locked_atomic("/a/b", 10, 10, true)
            .await
            .unwrap(),
        Some(false)
    );
    assert_eq!(
        locks
            .is_path_locked_atomic("/a/b", 10, 10, false)
            .await
            .unwrap(),
        Some(true)
    );
    // Decimal length changes must not reverse staleness.
    assert!(matches!(
        acquire(&locks, "/a/b", PathLockKind::Exact, "next", 10, 9).await,
        Err(PathLockError::Conflict { .. })
    ));
    acquire(&locks, "/a/b", PathLockKind::Exact, "next", 10, 10)
        .await
        .unwrap();
}

#[tokio::test]
async fn conflict_rolls_back_the_whole_batch_including_upgrades() {
    let dir = TempDir::new().unwrap();
    let locks = provider(&dir, "test");
    acquire(&locks, "/a", PathLockKind::Exact, "one", 1, 0)
        .await
        .unwrap();
    acquire(&locks, "/z", PathLockKind::Exact, "two", 1, 0)
        .await
        .unwrap();
    let before = locks.read_token("/a").await.unwrap();
    let result = locks
        .try_acquire_batch_atomic(
            &[
                request("/a", PathLockKind::Tree),
                request("/b", PathLockKind::Exact),
                request("/z", PathLockKind::Exact),
            ],
            "one",
            2,
            0,
        )
        .await;
    assert!(matches!(result, Err(PathLockError::Conflict { .. })));
    assert_eq!(locks.read_token("/a").await.unwrap(), before);
    assert_eq!(locks.read_token("/b").await.unwrap(), None);
}

#[tokio::test]
async fn rollback_restores_upgrade_and_preserves_reentrant_tokens() {
    let dir = TempDir::new().unwrap();
    let locks = provider(&dir, "test");
    acquire(&locks, "/a", PathLockKind::Exact, "one", 1, 0)
        .await
        .unwrap();
    let before = locks.read_token("/a").await.unwrap();
    let changes = locks
        .try_acquire_batch_atomic(
            &[
                request("/a", PathLockKind::Exact),
                request("/a", PathLockKind::Tree),
                request("/b", PathLockKind::Exact),
                request("/b", PathLockKind::Tree),
            ],
            "one",
            2,
            0,
        )
        .await
        .unwrap()
        .unwrap();
    assert_eq!(changes[0].change, AcquisitionChange::Reentrant);
    locks
        .rollback_acquisitions_atomic(&changes, "one")
        .await
        .unwrap();
    assert_eq!(locks.read_token("/a").await.unwrap(), before);
    assert_eq!(locks.read_token("/b").await.unwrap(), None);
}

#[tokio::test]
async fn changed_rollback_is_atomic_and_cas_checks_the_entire_token() {
    let dir = TempDir::new().unwrap();
    let locks = provider(&dir, "test");
    let changes = locks
        .try_acquire_batch_atomic(
            &[
                request("/a", PathLockKind::Exact),
                request("/b", PathLockKind::Exact),
            ],
            "one",
            1,
            0,
        )
        .await
        .unwrap()
        .unwrap();
    let expected = locks.read_token("/a").await.unwrap().unwrap();
    assert!(locks.refresh_token("/a", "one", 2).await.unwrap());
    assert!(!locks
        .compare_and_write_token("/a", &expected, &expected)
        .await
        .unwrap());
    assert!(locks
        .rollback_acquisitions_atomic(&changes, "one")
        .await
        .is_err());
    // /b was removed first, but the transaction must roll that deletion back.
    assert!(locks.read_token("/b").await.unwrap().is_some());
    let current = locks.read_token("/a").await.unwrap().unwrap();
    assert!(locks
        .compare_and_write_token("/a", &current, &expected)
        .await
        .unwrap());
    locks
        .rollback_acquisitions_atomic(&changes, "one")
        .await
        .unwrap();
    assert!(locks.scan_descendant_locks("/").await.unwrap().is_empty());
}

#[tokio::test]
async fn database_write_contention_is_retryable_busy() {
    let dir = TempDir::new().unwrap();
    let locks = provider(&dir, "test");
    let mut blocker = Connection::open(dir.path().join("locks.db")).unwrap();
    let tx = blocker
        .transaction_with_behavior(TransactionBehavior::Immediate)
        .unwrap();
    let start = Instant::now();
    assert!(matches!(
        acquire(&locks, "/a", PathLockKind::Tree, "one", 1, 0).await,
        Err(PathLockError::Busy { .. })
    ));
    assert!(start.elapsed() >= SQLITE_BUSY_TIMEOUT);
    assert!(start.elapsed() < Duration::from_secs(3));
    drop(tx);
    acquire(&locks, "/a", PathLockKind::Tree, "one", 1, 0)
        .await
        .unwrap();
}

#[tokio::test]
async fn run_retries_busy_from_clean_transactions_without_duplicate_mutations() {
    let dir = TempDir::new().unwrap();
    let locks = provider(&dir, "test");
    acquire(&locks, "/old", PathLockKind::Exact, "dead", 1, 0)
        .await
        .unwrap();
    let mut attempts = 0;
    let result = locks
        .run(move |conn, namespace| {
            attempts += 1;
            assert!(conn.is_autocommit());
            let tx = conn
                .transaction_with_behavior(TransactionBehavior::Immediate)
                .map_err(db_error)?;
            // Both mutations from each failed attempt must have rolled back.
            assert!(read(&tx, namespace, "/old")?.is_some());
            assert!(read(&tx, namespace, "/new")?.is_none());
            let removed = tx
                .execute(
                    "DELETE FROM pathlock_tokens WHERE namespace=?1 AND path='/old'",
                    params![namespace],
                )
                .map_err(db_error)?;
            write(
                &tx,
                namespace,
                "/new",
                &LockToken {
                    owner_id: "new".into(),
                    time_ns: 2,
                    lock_type: PathLockKind::Tree,
                },
            )?;
            if attempts < SQLITE_BUSY_ATTEMPTS {
                return Err(PathLockError::Busy {
                    lock_path: "sqlite".into(),
                    operation: "injected after mutation".into(),
                });
            }
            tx.commit().map_err(db_error)?;
            Ok((attempts, removed))
        })
        .await
        .unwrap();
    assert_eq!(result, (SQLITE_BUSY_ATTEMPTS, 1));
    assert!(locks.read_token("/old").await.unwrap().is_none());
    assert_eq!(
        locks.read_token("/new").await.unwrap().unwrap().owner_id,
        "new"
    );
}

#[tokio::test]
async fn run_bounds_busy_attempts_and_does_not_retry_other_errors() {
    use std::sync::atomic::{AtomicUsize, Ordering};
    let dir = TempDir::new().unwrap();
    let locks = provider(&dir, "test");
    for kind in ["busy", "conflict", "invalid", "io"] {
        let calls = Arc::new(AtomicUsize::new(0));
        let observed = calls.clone();
        let result: PathLockResult<()> = locks
            .run(move |_, _| {
                observed.fetch_add(1, Ordering::SeqCst);
                Err(match kind {
                    "busy" => PathLockError::Busy {
                        lock_path: "sqlite".into(),
                        operation: "injected".into(),
                    },
                    "conflict" => PathLockError::Conflict {
                        lock_path: "/a".into(),
                        owner: "other".into(),
                        kind: PathLockKind::Tree,
                    },
                    "invalid" => PathLockError::InvalidRequest("injected".into()),
                    _ => PathLockError::Io("injected".into()),
                })
            })
            .await;
        assert_eq!(
            calls.load(Ordering::SeqCst),
            if kind == "busy" {
                SQLITE_BUSY_ATTEMPTS
            } else {
                1
            }
        );
        assert!(matches!(
            (kind, result),
            ("busy", Err(PathLockError::Busy { .. }))
                | ("conflict", Err(PathLockError::Conflict { .. }))
                | ("invalid", Err(PathLockError::InvalidRequest(_)))
                | ("io", Err(PathLockError::Io(_)))
        ));
    }
}

async fn with_transient_writer<T>(
    blocker: &mut Connection,
    operation: impl std::future::Future<Output = T>,
) -> T {
    let tx = blocker
        .transaction_with_behavior(TransactionBehavior::Immediate)
        .unwrap();
    // Keep the writer longer than one busy timeout, then allow provider retries
    // to recover. Tokio's timer also verifies waits stay off the async executor.
    let (result, ()) = tokio::join!(operation, async {
        tokio::time::sleep(SQLITE_BUSY_TIMEOUT + Duration::from_millis(50)).await;
        drop(tx);
    });
    result
}

#[tokio::test]
async fn busy_retries_cover_zero_timeout_acquire_refresh_release_and_rollback() {
    let dir = TempDir::new().unwrap();
    let locks = Arc::new(provider(&dir, "test"));
    let manager = PathLockManager::new(
        Arc::new(MemFileSystem::new()),
        locks.clone(),
        PathLockConfig::default(),
    );
    let mut blocker = Connection::open(dir.path().join("locks.db")).unwrap();
    let lease = with_transient_writer(
        &mut blocker,
        manager.acquire_tree("/a", Duration::ZERO, None),
    )
    .await
    .unwrap();
    assert_eq!(
        with_transient_writer(&mut blocker, manager.refresh(&lease))
            .await
            .unwrap(),
        "refreshed"
    );
    assert!(with_transient_writer(
        &mut blocker,
        locks.refresh_token("/a", &lease.lease.owner_id, u128::MAX),
    )
    .await
    .unwrap());
    with_transient_writer(&mut blocker, manager.release(&lease))
        .await
        .unwrap();
    assert!(locks.read_token("/a").await.unwrap().is_none());

    let changes = acquire(&locks, "/rollback", PathLockKind::Tree, "owner", 1, 0)
        .await
        .unwrap();
    with_transient_writer(
        &mut blocker,
        locks.rollback_acquisitions_atomic(&changes, "owner"),
    )
    .await
    .unwrap();
    assert!(locks.read_token("/rollback").await.unwrap().is_none());
}

#[tokio::test]
async fn manager_zero_timeout_waits_for_transient_sqlite_writer() {
    let dir = TempDir::new().unwrap();
    let locks = Arc::new(provider(&dir, "test"));
    let manager = PathLockManager::new(
        Arc::new(MemFileSystem::new()),
        locks.clone(),
        PathLockConfig::default(),
    );
    let mut blocker = Connection::open(dir.path().join("locks.db")).unwrap();
    let tx = blocker
        .transaction_with_behavior(TransactionBehavior::Immediate)
        .unwrap();
    let (lease, ()) = tokio::join!(manager.acquire_tree("/a", Duration::ZERO, None), async {
        tokio::time::sleep(Duration::from_millis(20)).await;
        drop(tx);
    });
    let lease = lease.unwrap();
    manager.release(&lease).await.unwrap();
    assert!(locks.read_token("/a").await.unwrap().is_none());
}

#[tokio::test]
async fn rejects_invalid_paths_and_owner_before_mutation() {
    let dir = TempDir::new().unwrap();
    let locks = provider(&dir, "test");
    for path in ["relative", "/a/", "/a//b", "/a/../b", "/./a", "/a\0b"] {
        assert!(matches!(
            acquire(&locks, path, PathLockKind::Exact, "one", 1, 0).await,
            Err(PathLockError::InvalidRequest(_))
        ));
    }
    for owner in ["", "bad:owner"] {
        assert!(matches!(
            acquire(&locks, "/a", PathLockKind::Exact, owner, 1, 0).await,
            Err(PathLockError::InvalidRequest(_))
        ));
    }
    assert!(locks.scan_descendant_locks("/").await.unwrap().is_empty());
    assert!(SQLitePathLockProvider::new(":memory:", "test").is_err());
    assert!(
        SQLitePathLockProvider::new(dir.path().join("other.db").to_str().unwrap(), "").is_err()
    );
}

#[tokio::test]
async fn manager_lifecycle_uses_logical_handles_and_never_reads_business_directories() {
    let dir = TempDir::new().unwrap();
    let fs = Arc::new(MemFileSystem::new());
    let locks = Arc::new(provider(&dir, "test"));
    let manager = PathLockManager::new(fs.clone(), locks.clone(), PathLockConfig::default());
    let other = PathLockManager::new(
        fs.clone(),
        Arc::new(provider(&dir, "test")),
        PathLockConfig::default(),
    );
    // This business directory does not exist. A filesystem scan would fail.
    let exact = manager
        .acquire_exact("/missing/file", Duration::ZERO, None)
        .await
        .unwrap();
    assert_eq!(exact.lease.lock_paths, vec!["/missing/file"]);
    assert!(other
        .acquire_tree("/missing", Duration::ZERO, None)
        .await
        .is_err());
    let tree = manager
        .acquire_tree(
            "/missing",
            Duration::ZERO,
            Some((&exact.lease.lease_ref, &exact.ownership_ref)),
        )
        .await
        .unwrap();
    assert!(other.is_locked("/missing/child", true).await.unwrap());
    manager.release(&tree).await.unwrap();
    manager.release(&exact).await.unwrap();
    let tree = other
        .acquire_tree("/missing", Duration::ZERO, None)
        .await
        .unwrap();
    other.release(&tree).await.unwrap();
    assert!(locks.scan_descendant_locks("/").await.unwrap().is_empty());
    assert_eq!(
        fs.read_dir("/", None, None, None, None)
            .await
            .unwrap()
            .len(),
        0
    );
}

#[tokio::test]
async fn manager_upgrade_downgrades_and_handoff_adopts() {
    let dir = TempDir::new().unwrap();
    let locks = Arc::new(provider(&dir, "test"));
    let fs = Arc::new(MemFileSystem::new());
    let manager = PathLockManager::new(fs.clone(), locks.clone(), PathLockConfig::default());
    let exact = manager
        .acquire_exact("/a", Duration::ZERO, None)
        .await
        .unwrap();
    let tree = manager
        .acquire_tree(
            "/a",
            Duration::ZERO,
            Some((&exact.lease.lease_ref, &exact.ownership_ref)),
        )
        .await
        .unwrap();
    manager.release(&tree).await.unwrap();
    assert_eq!(
        locks.read_token("/a").await.unwrap().unwrap().lock_type,
        PathLockKind::Exact
    );
    let handoff = manager.to_handoff(&exact);
    manager.handoff(&exact).await.unwrap();
    let recipient = PathLockManager::new(
        fs,
        Arc::new(provider(&dir, "test")),
        PathLockConfig::default(),
    );
    let adopted = recipient.adopt(&handoff).await.unwrap();
    recipient.release(&adopted).await.unwrap();
    assert_eq!(locks.read_token("/a").await.unwrap(), None);
}

#[tokio::test]
async fn manager_refreshes_live_tokens() {
    let dir = TempDir::new().unwrap();
    let locks = Arc::new(provider(&dir, "test"));
    let config = PathLockConfig {
        lock_expire_secs: 1.0,
        ..PathLockConfig::default()
    };
    let manager = PathLockManager::new(Arc::new(MemFileSystem::new()), locks.clone(), config);
    let lease = manager
        .acquire_tree("/a", Duration::ZERO, None)
        .await
        .unwrap();
    let initial = locks.read_token("/a").await.unwrap().unwrap().time_ns;
    tokio::time::timeout(Duration::from_secs(3), async {
        loop {
            if locks.read_token("/a").await.unwrap().unwrap().time_ns > initial {
                break;
            }
            tokio::time::sleep(Duration::from_millis(20)).await;
        }
    })
    .await
    .unwrap();
    manager.release(&lease).await.unwrap();
}

#[tokio::test]
async fn stale_cleanup_is_bounded_and_keeps_this_owners_tokens() {
    let dir = TempDir::new().unwrap();
    let locks = provider(&dir, "test");
    let requests: Vec<_> = (0..300)
        .map(|id| request(&format!("/old/{id}"), PathLockKind::Exact))
        .collect();
    locks
        .try_acquire_batch_atomic(&requests, "old", 1, 0)
        .await
        .unwrap();
    acquire(&locks, "/own", PathLockKind::Exact, "new", 1, 0)
        .await
        .unwrap();
    acquire(&locks, "/new", PathLockKind::Exact, "new", 2, 2)
        .await
        .unwrap();
    assert_eq!(locks.scan_descendant_locks("/old").await.unwrap().len(), 44);
    assert!(locks.read_token("/own").await.unwrap().is_some());
    acquire(&locks, "/new", PathLockKind::Exact, "new", 3, 2)
        .await
        .unwrap();
    assert!(locks
        .scan_descendant_locks("/old")
        .await
        .unwrap()
        .is_empty());
}

#[tokio::test]
async fn stale_conflicting_descendants_cannot_be_revived_after_tree_takeover() {
    for root in ["/a", "/"] {
        let dir = TempDir::new().unwrap();
        let old = provider(&dir, "test");
        let new = provider(&dir, "test");
        let requests: Vec<_> = (0..300)
            .map(|id| {
                request(
                    &format!("{}/child/{id:03}", root.trim_end_matches('/')),
                    if id % 2 == 0 {
                        PathLockKind::Exact
                    } else {
                        PathLockKind::Tree
                    },
                )
            })
            .collect();
        old.try_acquire_batch_atomic(&requests, "old", 1, 0)
            .await
            .unwrap();
        let result = new
            .try_acquire_batch_atomic_with_stats(&[request(root, PathLockKind::Tree)], "new", 10, 2)
            .await
            .unwrap()
            .unwrap();
        let paths: Vec<_> = requests
            .iter()
            .map(|request| request.path.clone())
            .collect();
        let refreshed = old.refresh_tokens(&paths, "old", 11).await;
        assert!(
            refreshed.iter().all(|result| matches!(result, Ok(false))),
            "expired descendants must not revive under {root}: {refreshed:?}"
        );
        assert_eq!(result.stale_tokens_removed, 300);
        assert!(!old.refresh_token(&paths[299], "old", 12).await.unwrap());
        assert!(new.scan_descendant_locks(root).await.unwrap().is_empty());
        assert_eq!(new.read_token(root).await.unwrap().unwrap().owner_id, "new");
        assert!(matches!(
            acquire(&old, &paths[299], PathLockKind::Exact, "old", 12, 2).await,
            Err(PathLockError::Conflict {
                kind: PathLockKind::Tree,
                ..
            })
        ));
    }
}

#[tokio::test]
async fn stale_target_and_ancestor_trees_are_cleaned_beyond_unrelated_budget() {
    let dir = TempDir::new().unwrap();
    let old = provider(&dir, "test");
    let new = provider(&dir, "test");
    let unrelated: Vec<_> = (0..300)
        .map(|id| request(&format!("/unrelated/{id:03}"), PathLockKind::Exact))
        .collect();
    old.try_acquire_batch_atomic(&unrelated, "old", 0, 0)
        .await
        .unwrap();
    let conflicting = [
        request("/", PathLockKind::Tree),
        request("/a", PathLockKind::Tree),
        request("/a/b/c", PathLockKind::Exact),
    ];
    old.try_acquire_batch_atomic(&conflicting, "old", 1, 0)
        .await
        .unwrap();
    acquire(&old, "/a/b", PathLockKind::Exact, "old", 1, 0)
        .await
        .unwrap();
    // A separate namespace survives the cleanup.
    let other = provider(&dir, "other");
    acquire(&other, "/", PathLockKind::Tree, "old", 1, 0)
        .await
        .unwrap();
    let result = new
        .try_acquire_batch_atomic_with_stats(
            &[request("/a/b/c", PathLockKind::Exact)],
            "new",
            10,
            2,
        )
        .await
        .unwrap()
        .unwrap();
    assert_eq!(result.stale_tokens_removed, 259);
    let paths: Vec<_> = conflicting
        .iter()
        .map(|request| request.path.clone())
        .collect();
    assert!(old
        .refresh_tokens(&paths, "old", 11)
        .await
        .iter()
        .all(|result| matches!(result, Ok(false))));
    assert_eq!(
        old.scan_descendant_locks("/unrelated").await.unwrap().len(),
        44
    );
    assert!(old.refresh_token("/a/b", "old", 11).await.unwrap());
    assert!(other.refresh_token("/", "old", 11).await.unwrap());
}

#[tokio::test]
async fn failed_batch_restores_stale_conflicts_beyond_cleanup_budget() {
    let dir = TempDir::new().unwrap();
    let old = provider(&dir, "test");
    let new = provider(&dir, "test");
    let requests: Vec<_> = (0..300)
        .map(|id| request(&format!("/a/{id:03}"), PathLockKind::Exact))
        .collect();
    old.try_acquire_batch_atomic(&requests, "old", 1, 0)
        .await
        .unwrap();
    acquire(&old, "/live", PathLockKind::Exact, "live", 10, 0)
        .await
        .unwrap();
    assert!(matches!(
        new.try_acquire_batch_atomic_with_stats(
            &[
                request("/a", PathLockKind::Tree),
                request("/live", PathLockKind::Exact),
            ],
            "new",
            10,
            2,
        )
        .await,
        Err(PathLockError::Conflict { .. })
    ));
    assert!(new.read_token("/a").await.unwrap().is_none());
    assert_eq!(old.scan_descendant_locks("/a").await.unwrap().len(), 300);
    let paths: Vec<_> = requests
        .iter()
        .map(|request| request.path.clone())
        .collect();
    assert!(old
        .refresh_tokens(&paths, "old", 11)
        .await
        .iter()
        .all(|result| matches!(result, Ok(true))));
}

fn wait_for_file(path: &Path) {
    let start = Instant::now();
    while !path.exists() {
        assert!(
            start.elapsed() < Duration::from_secs(15),
            "timed out waiting for {path:?}"
        );
        std::thread::sleep(Duration::from_millis(5));
    }
}

#[tokio::test]
async fn batch_refresh_preserves_order_ownership_kinds_and_large_batches() {
    let dir = TempDir::new().unwrap();
    let locks = provider(&dir, "test");
    let requests: Vec<_> = (0..600)
        .map(|id| {
            request(
                &format!("/batch/{id}"),
                if id % 2 == 0 {
                    PathLockKind::Tree
                } else {
                    PathLockKind::Exact
                },
            )
        })
        .collect();
    locks
        .try_acquire_batch_atomic(&requests, "one", 1, 0)
        .await
        .unwrap();
    acquire(&locks, "/foreign", PathLockKind::Exact, "two", 1, 0)
        .await
        .unwrap();
    let mut paths: Vec<_> = requests
        .iter()
        .map(|request| request.path.clone())
        .collect();
    paths.extend([
        "/missing".into(),
        "/foreign".into(),
        "invalid".into(),
        "/batch/0".into(),
    ]);
    let results = locks.refresh_tokens(&paths, "one", u128::MAX).await;
    assert_eq!(results.len(), paths.len());
    assert!(results[..600]
        .iter()
        .all(|result| matches!(result, Ok(true))));
    assert!(matches!(results[600], Ok(false)));
    assert!(matches!(results[601], Ok(false)));
    assert!(matches!(
        results[602],
        Err(PathLockError::InvalidRequest(_))
    ));
    assert!(matches!(results[603], Ok(true)));
    for request in requests {
        let token = locks.read_token(&request.path).await.unwrap().unwrap();
        assert_eq!(token.time_ns, u128::MAX);
        assert_eq!(token.lock_type, request.kind);
    }
    assert_eq!(
        locks.read_token("/foreign").await.unwrap().unwrap().time_ns,
        1
    );
    assert!(locks.refresh_tokens(&[], "one", 2).await.is_empty());
}

#[tokio::test]
async fn batch_refresh_reports_busy_for_every_path_and_does_not_partially_commit() {
    let dir = TempDir::new().unwrap();
    let locks = provider(&dir, "test");
    acquire(&locks, "/a", PathLockKind::Exact, "one", 1, 0)
        .await
        .unwrap();
    acquire(&locks, "/b", PathLockKind::Exact, "one", 1, 0)
        .await
        .unwrap();
    let mut blocker = Connection::open(dir.path().join("locks.db")).unwrap();
    let tx = blocker
        .transaction_with_behavior(TransactionBehavior::Immediate)
        .unwrap();
    let results = locks
        .refresh_tokens(&["/a".into(), "/b".into()], "one", 2)
        .await;
    assert!(results
        .iter()
        .all(|result| matches!(result, Err(PathLockError::Busy { .. }))));
    drop(tx);
    assert_eq!(locks.read_token("/a").await.unwrap().unwrap().time_ns, 1);
    assert_eq!(locks.read_token("/b").await.unwrap().unwrap().time_ns, 1);
}

#[tokio::test]
async fn atomic_recovery_counts_committed_cleanup_and_stale_replacements_once() {
    let dir = TempDir::new().unwrap();
    let locks = provider(&dir, "test");
    let requests: Vec<_> = (0..300)
        .map(|id| request(&format!("/old/{id:03}"), PathLockKind::Exact))
        .collect();
    locks
        .try_acquire_batch_atomic(&requests, "old", 1, 0)
        .await
        .unwrap();
    let result = locks
        .try_acquire_batch_atomic_with_stats(
            &[request("/old/299", PathLockKind::Tree)],
            "new",
            10,
            2,
        )
        .await
        .unwrap()
        .unwrap();
    assert_eq!(result.stale_tokens_removed, 257);
    let result = locks
        .try_acquire_batch_atomic_with_stats(
            &[request("/old/299", PathLockKind::Tree)],
            "new",
            10,
            2,
        )
        .await
        .unwrap()
        .unwrap();
    assert_eq!(result.stale_tokens_removed, 43);
    let result = locks
        .try_acquire_batch_atomic_with_stats(
            &[request("/old/299", PathLockKind::Tree)],
            "new",
            10,
            2,
        )
        .await
        .unwrap()
        .unwrap();
    assert_eq!(result.stale_tokens_removed, 0);
}

#[tokio::test]
async fn manager_observes_only_committed_stale_recovery() {
    let dir = TempDir::new().unwrap();
    let locks = Arc::new(provider(&dir, "test"));
    acquire(&locks, "/old", PathLockKind::Exact, "dead", 1, 0)
        .await
        .unwrap();
    acquire(&locks, "/live", PathLockKind::Exact, "live", u128::MAX, 0)
        .await
        .unwrap();
    let manager = PathLockManager::new(
        Arc::new(MemFileSystem::new()),
        locks.clone(),
        PathLockConfig::default(),
    );
    assert!(manager
        .acquire_exact("/live", Duration::ZERO, None)
        .await
        .is_err());
    assert_eq!(manager.observe().await.stale_locks_removed, 0);
    assert!(locks.read_token("/old").await.unwrap().is_some());
    let lease = manager
        .acquire_tree("/old", Duration::ZERO, None)
        .await
        .unwrap();
    assert_eq!(manager.observe().await.stale_locks_removed, 1);
    manager.release(&lease).await.unwrap();
}

// Kill only subprocesses started by this test if a failed assertion unwinds.
struct Contender(Child);
impl Drop for Contender {
    fn drop(&mut self) {
        let _ = self.0.kill();
        let _ = self.0.wait();
    }
}

#[test]
fn separate_processes_cannot_acquire_tree_and_descendant_simultaneously() {
    let dir = TempDir::new().unwrap();
    drop(provider(&dir, "test"));
    for (left, right, successes) in [
        ("/a", "/a/b", 1),
        ("/", "/a/b", 1),
        ("/a", "/a", 1),
        ("/a", "/b", 2),
    ] {
        let round = dir.path().join(uuid::Uuid::new_v4().to_string());
        std::fs::create_dir(&round).unwrap();
        let mut children = Vec::new();
        for (id, path, kind) in [("left", left, "T"), ("right", right, "E")] {
            let child = Command::new(std::env::current_exe().unwrap())
                .args([
                    "--exact",
                    "lock::sqlite_provider::tests::subprocess_contender",
                    "--ignored",
                    "--nocapture",
                ])
                .env("OV_SQLITE_TEST_DIR", dir.path())
                .env("OV_SQLITE_TEST_ROUND", &round)
                .env("OV_SQLITE_TEST_ID", id)
                .env("OV_SQLITE_TEST_PATH", path)
                .env("OV_SQLITE_TEST_KIND", kind)
                .spawn()
                .unwrap();
            children.push(Contender(child));
        }
        wait_for_file(&round.join("left.ready"));
        wait_for_file(&round.join("right.ready"));
        std::fs::write(round.join("go"), "").unwrap();
        wait_for_file(&round.join("left.result"));
        wait_for_file(&round.join("right.result"));
        let results: Vec<_> = ["left", "right"]
            .iter()
            .map(|id| std::fs::read_to_string(round.join(format!("{id}.result"))).unwrap())
            .collect();
        std::fs::write(round.join("release"), "").unwrap();
        for child in &mut children {
            assert!(child.0.wait().unwrap().success());
        }
        assert_eq!(
            results
                .iter()
                .filter(|result| *result == "acquired")
                .count(),
            successes
        );
        assert_eq!(
            results
                .iter()
                .filter(|result| *result == "conflict")
                .count(),
            2 - successes
        );
    }
}

#[test]
fn crashed_process_token_is_persistent_and_can_be_reclaimed() {
    let dir = TempDir::new().unwrap();
    drop(provider(&dir, "test"));
    let round = dir.path().join("crash");
    std::fs::create_dir(&round).unwrap();
    std::fs::write(round.join("go"), "").unwrap();
    let mut child = Contender(
        Command::new(std::env::current_exe().unwrap())
            .args([
                "--exact",
                "lock::sqlite_provider::tests::subprocess_contender",
                "--ignored",
            ])
            .env("OV_SQLITE_TEST_DIR", dir.path())
            .env("OV_SQLITE_TEST_ROUND", &round)
            .env("OV_SQLITE_TEST_ID", "crashed")
            .env("OV_SQLITE_TEST_PATH", "/a/b")
            .env("OV_SQLITE_TEST_KIND", "E")
            .env("OV_SQLITE_TEST_CRASH", "1")
            .spawn()
            .unwrap(),
    );
    wait_for_file(&round.join("crashed.result"));
    assert!(child.0.wait().unwrap().success());
    let locks = provider(&dir, "test");
    let runtime = tokio::runtime::Runtime::new().unwrap();
    assert!(matches!(
        runtime.block_on(acquire(&locks, "/a", PathLockKind::Tree, "next", 101, 100)),
        Err(PathLockError::Conflict { .. })
    ));
    runtime
        .block_on(acquire(&locks, "/a", PathLockKind::Tree, "next", 101, 101))
        .unwrap();
    assert!(
        runtime
            .block_on(locks.remove_token("/a/b", "crashed", false))
            .unwrap()
            == false
    );
}

#[test]
#[ignore = "subprocess helper invoked by separate_processes test"]
fn subprocess_contender() {
    let dir = std::path::PathBuf::from(std::env::var("OV_SQLITE_TEST_DIR").unwrap());
    let round = std::path::PathBuf::from(std::env::var("OV_SQLITE_TEST_ROUND").unwrap());
    let id = std::env::var("OV_SQLITE_TEST_ID").unwrap();
    let path = std::env::var("OV_SQLITE_TEST_PATH").unwrap();
    let kind = if std::env::var("OV_SQLITE_TEST_KIND").unwrap() == "T" {
        PathLockKind::Tree
    } else {
        PathLockKind::Exact
    };
    let locks =
        SQLitePathLockProvider::new(dir.join("locks.db").to_str().unwrap(), "test").unwrap();
    let runtime = tokio::runtime::Runtime::new().unwrap();
    std::fs::write(round.join(format!("{id}.ready")), "").unwrap();
    wait_for_file(&round.join("go"));
    if std::env::var_os("OV_SQLITE_TEST_CRASH").is_some() {
        runtime
            .block_on(acquire(&locks, &path, kind, &id, 100, 0))
            .unwrap();
        std::fs::write(round.join(format!("{id}.result")), "acquired").unwrap();
        // Skip connection/runtime destructors as with an abruptly terminated owner.
        std::process::exit(0);
    }
    let manager = runtime.block_on(async {
        PathLockManager::new(
            Arc::new(MemFileSystem::new()),
            Arc::new(locks),
            PathLockConfig::default(),
        )
    });
    // Exercise the production path with the configured zero logical wait timeout.
    // No test-side Busy retry may hide a transient database contention failure.
    let lease = match runtime.block_on(manager.acquire_batch(
        &[request(&path, kind)],
        Duration::ZERO,
        None,
    )) {
        Ok(lease) => Some(lease),
        Err(PathLockError::Timeout { .. }) => {
            assert!(
                !runtime.block_on(manager.observe()).conflicts.is_empty(),
                "zero-timeout acquisition failed without an actual logical lock conflict"
            );
            None
        }
        Err(error) => panic!("{error}"),
    };
    std::fs::write(
        round.join(format!("{id}.result")),
        if lease.is_some() {
            "acquired"
        } else {
            "conflict"
        },
    )
    .unwrap();
    wait_for_file(&round.join("release"));
    if let Some(lease) = lease {
        runtime.block_on(manager.release(&lease)).unwrap();
    }
}