Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

13.8 KB
Raw Blame
//! Dispatch semantics: a long job does not hold the poll loop, a job is never
//! dispatched twice, single-flight skips are recorded, per-job retry budgets
//! are honoured and a system job's handler result is what gets recorded.

use super::*;
use crate::cron::system_job_handlers::{register, SystemJobContext};
use crate::cron::system_jobs::ensure_system_job;
use std::sync::atomic::{AtomicUsize, Ordering};
use tempfile::TempDir;
use tinyflows_bus::schedule::types::Schedule;
use tinyflows_bus::schedule::JobPolicy;

/// Stores `policy` on the job.
fn set_policy(config: &Config, job_id: &str, policy: JobPolicy) -> anyhow::Result<()> {
    cron::update_job(
        config,
        job_id,
        tinyflows_bus::schedule::types::CronJobPatch {
            policy: Some(policy),
            ..Default::default()
        },
    )
    .map(|_| ())
}

fn security(config: &Config) -> Arc<SecurityPolicy> {
    Arc::new(SecurityPolicy::from_config(
        &config.autonomy,
        &config.workspace_dir,
        &config.action_dir,
    ))
}

/// A system job row named `name`, already due.
fn due_system_job(config: &Config, name: &str) -> CronJob {
    let job = ensure_system_job(
        config,
        name,
        Schedule::Every {
            every_ms: 3_600_000,
        },
    )
    .unwrap();
    // Due now in the row the next poll reads.
    crate::cron::update_job(
        config,
        &job.id,
        tinyflows_bus::schedule::types::CronJobPatch {
            schedule: Some(Schedule::Every {
                every_ms: 3_600_000,
            }),
            ..Default::default()
        },
    )
    .unwrap();
    force_due(config, &job.id);
    // The runtime claims against the stored row, so hand back exactly that.
    cron::get_job(config, &job.id).unwrap()
}

fn force_due(config: &Config, job_id: &str) {
    crate::cron::store::tests::force_next_run_for_test(
        config,
        job_id,
        Utc::now() - ChronoDuration::seconds(1),
    )
    .unwrap();
}

fn counting(
    calls: Arc<AtomicUsize>,
    result: Result<(), String>,
) -> crate::cron::system_job_handlers::SystemJobHandler {
    Arc::new(move |_ctx: SystemJobContext| {
        calls.fetch_add(1, Ordering::SeqCst);
        let result = result.clone();
        Box::pin(async move { result })
    })
}

#[tokio::test]
async fn zero_retries_means_exactly_one_attempt() {
    let tmp = TempDir::new().unwrap();
    let mut config = test_config(&tmp).await;
    config.reliability.scheduler_retries = 2;
    config.reliability.provider_backoff_ms = 200;
    let calls = Arc::new(AtomicUsize::new(0));
    let _handler = register(
        "dispatch-once",
        counting(Arc::clone(&calls), Err("boom".to_string())),
    );
    let job = due_system_job(&config, "dispatch-once");
    set_policy(
        &config,
        &job.id,
        JobPolicy {
            retries: Some(0),
            single_flight: false,
        },
    )
    .unwrap();
    let job = cron::get_job(&config, &job.id).unwrap();
    let (success, output) = execute_job_now(
        &crate::core::runtime::DispatchCtx::ambient(),
        &config,
        &job,
        "dispatch-run",
    )
    .await;
    assert!(!success);
    assert!(output.contains("boom"), "{output}");
    assert_eq!(calls.load(Ordering::SeqCst), 1);
}

#[tokio::test]
async fn without_a_policy_the_configured_retry_budget_applies() {
    let tmp = TempDir::new().unwrap();
    let mut config = test_config(&tmp).await;
    config.reliability.scheduler_retries = 1;
    config.reliability.provider_backoff_ms = 200;
    let calls = Arc::new(AtomicUsize::new(0));
    let _handler = register(
        "dispatch-default-retries",
        counting(Arc::clone(&calls), Err("again".to_string())),
    );
    let job = due_system_job(&config, "dispatch-default-retries");
    let (success, _) = execute_job_now(
        &crate::core::runtime::DispatchCtx::ambient(),
        &config,
        &job,
        "dispatch-run",
    )
    .await;
    assert!(!success);
    assert_eq!(calls.load(Ordering::SeqCst), 2);
}

#[tokio::test]
async fn a_system_job_handler_error_is_recorded_as_the_run_result() {
    let tmp = TempDir::new().unwrap();
    let config = test_config(&tmp).await;
    let _handler = register(
        "dispatch-recorded",
        counting(
            Arc::new(AtomicUsize::new(0)),
            Err("handler failed".to_string()),
        ),
    );
    let job = due_system_job(&config, "dispatch-recorded");
    set_policy(
        &config,
        &job.id,
        JobPolicy {
            retries: Some(0),
            single_flight: false,
        },
    )
    .unwrap();
    let job = cron::get_job(&config, &job.id).unwrap();
    let mut dispatcher = JobDispatcher::new(
        &crate::core::runtime::DispatchCtx::ambient(),
        &config,
        &security(&config),
    );
    dispatcher.tick(&config).await.unwrap();
    dispatcher.drain().await;
    let stored = cron::get_job(&config, &job.id).unwrap();
    assert_eq!(stored.last_status.as_deref(), Some("error"));
    assert!(stored
        .last_output
        .unwrap_or_default()
        .contains("handler failed"));
}

#[tokio::test]
async fn a_system_job_without_a_handler_is_still_ok_on_dispatch() {
    let tmp = TempDir::new().unwrap();
    let config = test_config(&tmp).await;
    let job = due_system_job(&config, "dispatch-unclaimed");
    let (success, output) = execute_job_now(
        &crate::core::runtime::DispatchCtx::ambient(),
        &config,
        &job,
        "dispatch-run",
    )
    .await;
    assert!(success);
    assert!(output.contains("dispatched"), "{output}");
}

#[tokio::test]
async fn module_scheduler_slot_is_per_store_scope_and_refuses_a_duplicate() {
    let first_tmp = TempDir::new().unwrap();
    let first_config = test_config(&first_tmp).await;
    let mut first = JobDispatcher::new(
        &crate::core::runtime::DispatchCtx::ambient(),
        &first_config,
        &security(&first_config),
    );
    assert_eq!(first.tick(&first_config).await.unwrap(), 0);

    let mut duplicate = JobDispatcher::new(
        &crate::core::runtime::DispatchCtx::ambient(),
        &first_config,
        &security(&first_config),
    );
    let error = duplicate
        .tick(&first_config)
        .await
        .expect_err("the same document scope has one module scheduler");
    assert!(
        error
            .to_string()
            .contains("scheduler already owns this scope"),
        "unexpected duplicate-scope refusal: {error}"
    );

    let second_tmp = TempDir::new().unwrap();
    let second_config = test_config(&second_tmp).await;
    let mut second = JobDispatcher::new(
        &crate::core::runtime::DispatchCtx::ambient(),
        &second_config,
        &security(&second_config),
    );
    assert_eq!(
        second.tick(&second_config).await.unwrap(),
        0,
        "a different document/workspace scope owns an independent module slot"
    );
    first.drain().await;
    second.drain().await;
}

#[tokio::test]
async fn started_agent_scheduler_releases_its_context_after_stop_is_confirmed() {
    use crate::core::runtime::{ContextOverlay, CoreContext, DispatchCtx, DomainSet};

    let tmp = TempDir::new().unwrap();
    let config = test_config(&tmp).await;
    let agent = format!("retire-{}", uuid::Uuid::new_v4().simple());
    let context = CoreContext::for_test_with_config(DomainSet::full(), config.clone()).derive_with(
        ContextOverlay::new(
            config.clone(),
            DomainSet::full(),
            crate::tools::toolpacks::ToolGroups::none(),
        )
        .session_agent(agent),
    );
    let weak = Arc::downgrade(&context);
    let mut dispatcher = JobDispatcher::new(
        &DispatchCtx::new(Arc::clone(&context)),
        &config,
        &security(&config),
    );
    assert_eq!(dispatcher.tick(&config).await.unwrap(), 0);

    drop(dispatcher);
    drop(context);
    tokio::time::timeout(std::time::Duration::from_secs(3), async {
        while weak.upgrade().is_some() {
            tokio::task::yield_now().await;
        }
    })
    .await
    .expect("the finite StopScheduler owner releases its context after confirmation");
}

#[cfg(feature = "legacy-import")]
/// Upgrade regression: a job whose retry budget and single-flight setting lived
/// in the old `cron_job_policies` side table keeps firing after the upgrade,
/// with that policy now on the job itself.
#[tokio::test]
async fn an_upgraded_job_with_a_side_table_policy_keeps_firing_with_it() {
    let tmp = TempDir::new().unwrap();
    let config = test_config(&tmp).await;
    let legacy = crate::storage::import::tinyflows::legacy_jobs_db(
        &crate::core::runtime::agent_scope_dir(&config),
    );
    std::fs::create_dir_all(legacy.parent().unwrap()).unwrap();
    std::fs::copy(
        std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
            .join("src/storage/import/fixtures/tinyflows/workspace/cron/jobs.db"),
        &legacy,
    )
    .unwrap();
    let shell_id = "a0ebbb67-f060-4467-92f7-78d6272be94e";
    {
        let conn = crate::storage::rusqlite::Connection::open(&legacy).unwrap();
        conn.execute_batch(
            "CREATE TABLE IF NOT EXISTS cron_job_policies (
                 job_id TEXT PRIMARY KEY, retries INTEGER, single_flight INTEGER NOT NULL DEFAULT 0)",
        )
        .unwrap();
        conn.execute(
            "INSERT INTO cron_job_policies (job_id, retries, single_flight) VALUES (?1, 0, 1)",
            [shell_id],
        )
        .unwrap();
    }

    // The storage importer brings the legacy rows (and their side-table policy) over.
    let backend = crate::storage::local::backend_for(&config.workspace_dir).unwrap();
    for report in crate::storage::import::tinyflows::run_with(&backend, &config) {
        report.outcome.unwrap();
    }
    force_due(&config, shell_id);
    let job = cron::get_job(&config, shell_id).unwrap();
    assert_eq!(job.policy.retries, Some(0));
    assert!(job.policy.single_flight);

    let mut dispatcher = JobDispatcher::new(
        &crate::core::runtime::DispatchCtx::ambient(),
        &config,
        &security(&config),
    );
    dispatcher.tick(&config).await.unwrap();
    dispatcher.drain().await;

    let after = cron::get_job(&config, shell_id).unwrap();
    assert_eq!(
        after.last_status.as_deref(),
        Some("ok"),
        "the upgraded job ran"
    );
    assert!(after.policy.single_flight, "its policy survived the run");
}

/// A `flow` schedule row fires through the runtime's trigger sink: the run is
/// recorded without any domain event in between, and an unknown
/// flow is a no-op rather than a failure.
#[tokio::test]
async fn a_flow_schedule_job_fires_through_the_trigger_sink() {
    let tmp = TempDir::new().unwrap();
    let config = test_config(&tmp).await;
    let job = cron::add_flow_schedule_job(
        &config,
        "flow-missing-for-sink",
        Schedule::Every {
            every_ms: 3_600_000,
        },
    )
    .unwrap();
    force_due(&config, &job.id);
    let job = cron::get_job(&config, &job.id).unwrap();

    let mut dispatcher = JobDispatcher::new(
        &crate::core::runtime::DispatchCtx::ambient(),
        &config,
        &security(&config),
    );
    dispatcher.tick(&config).await.unwrap();
    dispatcher.drain().await;

    let stored = cron::get_job(&config, &job.id).unwrap();
    assert_eq!(stored.last_status.as_deref(), Some("ok"));
    assert!(stored
        .last_output
        .unwrap_or_default()
        .contains("flow schedule tick emitted for flow flow-missing-for-sink"));
    assert!(stored.next_run > Utc::now(), "the recurrence moved on");
}

/// `[components] flows = false` stops schedule-triggered flow rows (the tick is
/// answered as skipped and the recurrence moves on) while the row stays valid.
#[tokio::test]
async fn a_flow_schedule_job_is_skipped_when_the_flows_component_is_off() {
    use crate::core::runtime::{ComponentSet, CoreContext};

    let tmp = TempDir::new().unwrap();
    let config = test_config(&tmp).await;
    let job = cron::add_flow_schedule_job(
        &config,
        "flow-off-for-sink",
        Schedule::Every {
            every_ms: 3_600_000,
        },
    )
    .unwrap();
    force_due(&config, &job.id);
    let job = cron::get_job(&config, &job.id).unwrap();
    let job_id = job.id.clone();

    let mut off = ComponentSet::all();
    off.flows = false;
    let ctx = CoreContext::for_test_with_components(off);
    let security = security(&config);
    let cfg = config.clone();
    let dispatch = crate::core::runtime::DispatchCtx::new(Arc::clone(&ctx));
    CoreContext::scope(ctx, async move {
        let mut dispatcher = JobDispatcher::new(&dispatch, &cfg, &security);
        dispatcher.tick(&cfg).await.unwrap();
        dispatcher.drain().await;
    })
    .await;

    let stored = cron::get_job(&config, &job_id).unwrap();
    let output = stored.last_output.unwrap_or_default();
    assert!(output.contains("COMPONENT_DISABLED:flows"), "got: {output}");
    assert!(!output.contains("tick emitted"));
    assert!(stored.next_run > Utc::now(), "the recurrence moved on");
}

#[tokio::test]
async fn a_dispatcher_binds_the_context_it_is_handed_not_the_ambient_one() {
    let tmp = TempDir::new().unwrap();
    let config = test_config(&tmp).await;
    let security = security(&config);
    let handed =
        crate::core::runtime::CoreContext::for_test(crate::core::runtime::DomainSet::full(), None);
    let ambient =
        crate::core::runtime::CoreContext::for_test(crate::core::runtime::DomainSet::full(), None);

    let dispatcher = crate::core::runtime::CoreContext::scope(Arc::clone(&ambient), async {
        // Across an await the handed context is still the one bound.
        tokio::task::yield_now().await;
        JobDispatcher::new(
            &crate::core::runtime::DispatchCtx::new(Arc::clone(&handed)),
            &config,
            &security,
        )
    })
    .await;
    let bound = dispatcher.dispatch_ctx();
    assert!(Arc::ptr_eq(bound.core().expect("bound"), &handed));
    assert!(!Arc::ptr_eq(bound.core().unwrap(), &ambient));
}