Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

9.1 KB
Raw Blame
//! Priority ordering, per-origin admission and ordered HTTP completions.

use std::{
    collections::{HashMap, VecDeque},
    hash::Hash,
    num::NonZeroUsize,
    time::{Duration, Instant},
};

use curl::{easy::Handler, multi::Easy2Handle};

use super::{CurlMultiJob, CurlOriginKey, CurlTransferId};

pub(super) struct CurlActiveTransfer<H: Handler, C> {
    pub(super) handle: Easy2Handle<H>,
    pub(super) context: C,
    pub(super) origin: Option<CurlOriginKey>,
    pub(super) priority: u8,
    pub(super) label: String,
    pub(super) started_at: Instant,
    pub(super) queued_for: Duration,
}

pub(super) struct CurlPendingJob<H: Handler, C> {
    pub(super) transfer_id: CurlTransferId,
    pub(super) job: CurlMultiJob<H, C>,
    pub(super) enqueued_at: Instant,
}

impl<H: Handler, C> CurlPendingJob<H, C> {
    pub(super) fn deadline_reached(&self, now: Instant) -> bool {
        self.job.deadline.is_some_and(|deadline| deadline <= now)
    }
}

pub(super) fn take_expired_pending_jobs<H: Handler, C>(
    pending: &mut VecDeque<CurlPendingJob<H, C>>,
    now: Instant,
) -> Vec<CurlPendingJob<H, C>> {
    let mut retained = VecDeque::with_capacity(pending.len());
    let mut expired = Vec::new();
    while let Some(job) = pending.pop_front() {
        if job.deadline_reached(now) {
            expired.push(job);
        } else {
            retained.push_back(job);
        }
    }
    *pending = retained;
    expired
}

pub(super) fn enqueue_pending_job<H: Handler, C>(
    pending: &mut VecDeque<CurlPendingJob<H, C>>,
    transfer_id: CurlTransferId,
    job: CurlMultiJob<H, C>,
) {
    enqueue_existing_pending_job(
        pending,
        CurlPendingJob {
            transfer_id,
            job,
            enqueued_at: Instant::now(),
        },
    );
}

pub(super) fn enqueue_existing_pending_job<H: Handler, C>(
    pending: &mut VecDeque<CurlPendingJob<H, C>>,
    pending_job: CurlPendingJob<H, C>,
) {
    if let Some(index) = pending
        .iter()
        .position(|queued| pending_job.job.priority > queued.job.priority)
    {
        pending.insert(index, pending_job);
    } else {
        pending.push_back(pending_job);
    }
}

pub(super) fn job_is_eligible<H: Handler, C>(
    origin: Option<&CurlOriginKey>,
    active: &HashMap<CurlTransferId, CurlActiveTransfer<H, C>>,
    max_active_per_host: Option<NonZeroUsize>,
) -> bool {
    match (origin, max_active_per_host) {
        (Some(origin), Some(limit)) => active_origin_count(active, origin) < limit.get(),
        _ => true,
    }
}

pub(super) fn active_origin_count<H: Handler, C>(
    active: &HashMap<CurlTransferId, CurlActiveTransfer<H, C>>,
    origin: &CurlOriginKey,
) -> usize {
    active
        .values()
        .filter(|active| active.origin.as_ref() == Some(origin))
        .count()
}

pub(super) fn pending_origin_count<H: Handler, C>(
    pending: &VecDeque<CurlPendingJob<H, C>>,
    origin: &CurlOriginKey,
) -> usize {
    pending
        .iter()
        .filter(|pending| pending.job.origin.as_ref() == Some(origin))
        .count()
}

/// Removes exact active transfers while preserving libcurl's notification
/// order. Unknown IDs are stale terminals for already-retired residences and
/// cannot recover or disturb a newer transfer.
pub(super) fn take_transfers_in_notification_order<K, T, E>(
    active: &mut HashMap<K, T>,
    completed: Vec<(K, E)>,
) -> Vec<(K, T, E)>
where
    K: Copy + Eq + Hash,
{
    completed
        .into_iter()
        .filter_map(|(transfer_id, result)| {
            active
                .remove(&transfer_id)
                .map(|transfer| (transfer_id, transfer, result))
        })
        .collect()
}

#[cfg(test)]
mod tests {
    use curl::{easy::Easy2, multi::Multi};

    use crate::dns_adapter::CurlDnsResolution;

    use super::*;

    #[derive(Debug)]
    struct TestHandler;

    impl Handler for TestHandler {}

    fn test_job(
        label: &str,
        priority: u8,
        origin: Option<CurlOriginKey>,
    ) -> CurlMultiJob<TestHandler, String> {
        CurlMultiJob {
            easy: Easy2::new(TestHandler),
            context: label.to_owned(),
            origin,
            deadline: None,
            dns_resolution: CurlDnsResolution::no_shared_resolution(),
            priority,
            label: label.to_owned(),
        }
    }

    fn test_origin(host: &str) -> CurlOriginKey {
        CurlOriginKey {
            scheme: "https".to_owned(),
            host: host.to_owned(),
            port: Some(443),
        }
    }

    fn test_transfer_id(sequence: usize) -> CurlTransferId {
        CurlTransferId::from_token(sequence).expect("test transfer ID is non-zero")
    }

    #[test]
    fn pending_jobs_are_ordered_by_priority() {
        let mut pending = VecDeque::new();

        enqueue_pending_job(
            &mut pending,
            test_transfer_id(1),
            test_job("auto-a", 1, None),
        );
        enqueue_pending_job(&mut pending, test_transfer_id(2), test_job("low", 0, None));
        enqueue_pending_job(
            &mut pending,
            test_transfer_id(3),
            test_job("high-a", 2, None),
        );
        enqueue_pending_job(
            &mut pending,
            test_transfer_id(4),
            test_job("auto-b", 1, None),
        );
        enqueue_pending_job(
            &mut pending,
            test_transfer_id(5),
            test_job("high-b", 2, None),
        );

        let ordered = pending
            .iter()
            .map(|job| (job.transfer_id.token(), job.job.label.as_str()))
            .collect::<Vec<_>>();
        assert_eq!(
            ordered,
            [
                (3, "high-a"),
                (5, "high-b"),
                (1, "auto-a"),
                (4, "auto-b"),
                (2, "low"),
            ]
        );
    }

    #[test]
    fn expired_pending_jobs_are_removed_without_reordering_live_jobs() {
        let now = Instant::now();
        let mut expired = test_job("expired", 2, None);
        expired.deadline = Some(now);
        let mut live_high = test_job("live-high", 2, None);
        live_high.deadline = now.checked_add(Duration::from_secs(1));
        let live_low = test_job("live-low", 1, None);
        let mut pending = VecDeque::new();
        enqueue_pending_job(&mut pending, test_transfer_id(1), expired);
        enqueue_pending_job(&mut pending, test_transfer_id(2), live_high);
        enqueue_pending_job(&mut pending, test_transfer_id(3), live_low);

        let expired = take_expired_pending_jobs(&mut pending, now);

        assert_eq!(
            expired
                .iter()
                .map(|pending| pending.job.label.as_str())
                .collect::<Vec<_>>(),
            ["expired"]
        );
        assert_eq!(
            pending
                .iter()
                .map(|pending| pending.job.label.as_str())
                .collect::<Vec<_>>(),
            ["live-high", "live-low"]
        );
    }

    #[test]
    fn completed_jobs_preserve_libcurl_notification_order() {
        let mut active = HashMap::from([
            (test_transfer_id(1), "first"),
            (test_transfer_id(2), "second"),
            (test_transfer_id(3), "third"),
            (test_transfer_id(4), "fourth"),
        ]);

        // libcurl reported the newest transfer first, followed by the oldest.
        // Hash-map storage must not leak its iteration order to completions.
        let completed = take_transfers_in_notification_order(
            &mut active,
            vec![
                (test_transfer_id(4), "fourth-result"),
                (test_transfer_id(1), "first-result"),
            ],
        );

        assert_eq!(
            completed,
            vec![
                (test_transfer_id(4), "fourth", "fourth-result"),
                (test_transfer_id(1), "first", "first-result"),
            ]
        );
        assert_eq!(active.len(), 2);
        assert_eq!(active[&test_transfer_id(2)], "second");
        assert_eq!(active[&test_transfer_id(3)], "third");
    }

    #[test]
    fn stale_completion_cannot_remove_a_live_transfer() {
        let mut active = HashMap::from([(test_transfer_id(2), "live")]);

        let completed = take_transfers_in_notification_order(
            &mut active,
            vec![(test_transfer_id(1), "stale-result")],
        );

        assert!(completed.is_empty());
        assert_eq!(active[&test_transfer_id(2)], "live");
    }

    #[test]
    fn per_origin_cap_blocks_only_matching_origin() {
        let capped_origin = test_origin("example.test");
        let other_origin = test_origin("other.test");
        let multi = Multi::new();
        let active = CurlActiveTransfer {
            handle: multi
                .add2(Easy2::new(TestHandler))
                .expect("test handle should add to multi"),
            context: "active".to_owned(),
            origin: Some(capped_origin.clone()),
            priority: 1,
            label: "active".to_owned(),
            started_at: Instant::now(),
            queued_for: Duration::ZERO,
        };
        let active = HashMap::from([(test_transfer_id(1), active)]);
        let cap = NonZeroUsize::new(1);

        assert!(!job_is_eligible(Some(&capped_origin), &active, cap));
        assert!(job_is_eligible(Some(&other_origin), &active, cap));
    }
}