Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

12.8 KB
Raw Blame
//! Host wiring for the context ladder. The ladder's order and sizing rules live
//! in `tinyagents_harness::middleware::ContextLadder`; this file supplies only
//! what the host owns: the summarizer behind compression (task-state or free
//! form, with memory recall), the wrap-up prompts and deliverable tools, and
//! the outcome sink the wrap-up restores from.

use std::sync::Arc;

use async_trait::async_trait;
use tinyagents_harness::middleware::{
    compression_policy, legacy_max_input_tokens, microcompact_for_window, split_input_allowance,
    ArtifactIndexTocMiddleware, CapturedOutcomes, ContextCompressionMiddleware, ContextLadder,
    FinalCallWrapUpMiddleware, ImageAwareMessageTrimMiddleware, OutcomesUnavailable,
};
use tinyagents_harness::runtime::AgentHarness;
use tinyagents_harness::summarization::{
    FaultTolerantCachingSummarizer, ModelSummarizer, SummarizationPolicy, Summarizer,
    SummaryRecord, SummaryRequest, TaskStateSummarizer, DEFAULT_TASK_STATE_CHUNK_TOKENS,
};
use tinyinference_llm::message::Message;

use crate::agent::tinyagents::host::OpenHumanRunContext;
use crate::agent::tinyagents::memory_summarizer::MemoryRecallSummarizer;
use tinyagents_harness::turn_outcome::ToolOutcomeSink;

/// Fractions of a capped turn's model-call budget at which the model is told
/// how many calls are left.
const BUDGET_NOTICE_THRESHOLDS: [f64; 2] = [0.5, 0.8];
/// Longest verbatim tail of a task-state compaction (tokens).
const TASK_STATE_KEEP_RECENT_TOKENS: u64 = 20_000;
/// Store the artifact index is registered under on the run context.
const ARTIFACT_INDEX_STORE: &str =
    crate::agent::harness::tool_result_artifacts::TINYAGENTS_TOOL_RESULT_ARTIFACT_STORE;

/// The model a compaction's summarizer calls, and how it calls it.
pub(crate) struct CompactionSummarizer {
    /// The model the summary request goes to.
    pub(crate) model: Arc<dyn tinyinference_llm::model::ChatModel<()>>,
    /// The id `model` calls, for the summarizer's logs and summary provenance.
    pub(crate) model_id: String,
    /// `[context] summarizer_tool_rendering` / `summarizer_max_output_tokens`;
    /// the free-form (`Summary`) strategy applies them.
    pub(crate) settings: crate::config::SummarizerSettings,
}

/// Called when the turn starts summarizing earlier conversation.
pub(super) type CompactionNotice = Arc<dyn Fn() + Send + Sync>;

/// A summarizer that reports each summarization before it runs, so the client
/// can say what a turn is doing while a long compaction holds its next model
/// call. It sits under the cache, so a cached summary is not reported.
struct AnnouncingSummarizer {
    inner: Box<dyn Summarizer>,
    notice: CompactionNotice,
}

#[async_trait]
impl Summarizer for AnnouncingSummarizer {
    async fn summarize(
        &self,
        messages: &[Message],
    ) -> tinyagents_harness::error::Result<SummaryRecord> {
        (self.notice)();
        self.inner.summarize(messages).await
    }

    async fn summarize_request(
        &self,
        request: &SummaryRequest,
    ) -> tinyagents_harness::error::Result<SummaryRecord> {
        (self.notice)();
        self.inner.summarize_request(request).await
    }

    async fn merge(
        &self,
        summaries: &[SummaryRecord],
    ) -> tinyagents_harness::error::Result<SummaryRecord> {
        self.inner.merge(summaries).await
    }
}

/// `inner`, reporting each summarization to `notice` when there is one.
fn announced(inner: Box<dyn Summarizer>, notice: Option<CompactionNotice>) -> Box<dyn Summarizer> {
    match notice {
        Some(notice) => Box::new(AnnouncingSummarizer { inner, notice }),
        None => inner,
    }
}

/// Read side of the tool-outcome capture sink, for the wrap-up's restoration.
pub(super) struct OutcomeSinkSource(pub(super) ToolOutcomeSink);

impl CapturedOutcomes for OutcomeSinkSource {
    fn content_for(&self, call_id: &str) -> Result<Option<String>, OutcomesUnavailable> {
        let outcomes = self.0.lock().map_err(|_| OutcomesUnavailable)?;
        Ok(outcomes
            .iter()
            .find(|o| o.call_id == call_id)
            .map(|o| o.content.clone()))
    }
}

/// Install the ladder; returns the compression and wrap-up handles the run loop
/// reads after the drive future returns. `wrap_up_at_cap` is true only for a
/// top-level turn that pauses at its model-call cap.
#[allow(clippy::too_many_arguments)]
pub(super) fn install_context_ladder(
    harness: &mut AgentHarness<(), OpenHumanRunContext>,
    model: &str,
    context_window: Option<u64>,
    autocompact_enabled: bool,
    compaction_trigger_tokens: Option<u64>,
    compaction_strategy: crate::config::CompactionStrategy,
    microcompact_keep_recent: usize,
    summarizer: CompactionSummarizer,
    wrap_up_at_cap: bool,
    tool_outcome_sink: &ToolOutcomeSink,
    memory_turn: Option<Arc<crate::memory::lifecycle::hooks::MemoryTurn>>,
    compaction_notice: Option<CompactionNotice>,
) -> (
    Option<Arc<ContextCompressionMiddleware>>,
    Option<Arc<FinalCallWrapUpMiddleware>>,
) {
    let window = context_window.filter(|w| *w > 0);
    let compression = autocompact_enabled
        .then(|| compression_policy(context_window, compaction_trigger_tokens))
        .flatten()
        .map(|policy| {
            Arc::new(compression_middleware(
                policy,
                compaction_strategy,
                context_window,
                summarizer,
                model,
                memory_turn,
                compaction_notice,
            ))
        });
    // One allowance split between the contents list and wrap-up restoration.
    let trim_allowance = window
        .map(|w| legacy_max_input_tokens(w).max(1))
        .unwrap_or(0);
    let (toc_allowance, restore_allowance) = split_input_allowance(trim_allowance);
    let wrap_up = wrap_up_at_cap.then(|| {
        Arc::new(
            FinalCallWrapUpMiddleware::new(
                crate::agent::session_host::product_copy::OPENHUMAN_CLOSE_COPY
                    .max_iteration_instruction,
                crate::agent::session_host::product_copy::OPENHUMAN_CLOSE_COPY
                    .final_write_instruction,
                Arc::new(OutcomeSinkSource(tool_outcome_sink.clone())),
                restore_allowance,
            )
            .with_deliverable_tools(["file_write", "apply_patch"])
            .with_cleared_placeholder(
                tinyagents_harness::middleware::library::DEFAULT_CLEARED_PLACEHOLDER,
            )
            .with_budget_notice(BUDGET_NOTICE_THRESHOLDS),
        )
    });
    ContextLadder {
        compression,
        microcompact: (microcompact_keep_recent > 0).then(|| {
            microcompact_for_window(
                microcompact_keep_recent,
                tinyagents_harness::middleware::library::DEFAULT_CLEARED_PLACEHOLDER,
                context_window,
            )
        }),
        wrap_up,
        artifact_toc: Some(ArtifactIndexTocMiddleware::new(
            toc_allowance,
            ARTIFACT_INDEX_STORE,
        )),
        trim: window.map(ImageAwareMessageTrimMiddleware::for_context_window),
    }
    .install(harness)
}

/// The compression middleware for `strategy`: `TaskState` writes a typed
/// checkpoint over a token-budgeted tail, `Summary` is the free-form summarizer,
/// rendering tool activity and capping its output as `summarizer.settings`
/// say. `model` is the turn's.
pub(crate) fn compression_middleware(
    policy: SummarizationPolicy,
    strategy: crate::config::CompactionStrategy,
    context_window: Option<u64>,
    summarizer: CompactionSummarizer,
    model: &str,
    memory_turn: Option<Arc<crate::memory::lifecycle::hooks::MemoryTurn>>,
    compaction_notice: Option<CompactionNotice>,
) -> ContextCompressionMiddleware {
    let window = context_window.filter(|w| *w > 0);
    match strategy {
        crate::config::CompactionStrategy::TaskState => {
            let keep_recent = window.map_or(TASK_STATE_KEEP_RECENT_TOKENS, |w| {
                (w / 5).min(TASK_STATE_KEEP_RECENT_TOKENS)
            });
            let chunk = window.map_or(DEFAULT_TASK_STATE_CHUNK_TOKENS, |w| {
                (w * 2 / 5).clamp(4_000, DEFAULT_TASK_STATE_CHUNK_TOKENS)
            });
            tracing::info!(model, keep_recent_tokens = keep_recent, max_chunk_tokens = chunk, context_window = ?window, "[context_compression] strategy=task_state");
            let summarizer = FaultTolerantCachingSummarizer::new(
                announced(
                    Box::new(
                        TaskStateSummarizer::new(summarizer.model, summarizer.model_id)
                            .with_max_chunk_tokens(chunk),
                    ),
                    compaction_notice,
                ),
                &policy,
            );
            ContextCompressionMiddleware::with_summarizer(
                policy,
                MemoryRecallSummarizer::wrap(Box::new(summarizer), memory_turn),
            )
            .with_keep_recent_tokens(keep_recent)
        }
        crate::config::CompactionStrategy::Summary => {
            let settings = summarizer.settings;
            tracing::info!(
                model,
                tool_rendering = ?settings.tool_rendering,
                max_output_tokens = ?settings.max_output_tokens,
                "[context_compression] strategy=summary"
            );
            let mut model_summarizer = ModelSummarizer::new(summarizer.model, summarizer.model_id)
                .with_tool_rendering(settings.tool_rendering);
            if let Some(cap) = settings.max_output_tokens {
                model_summarizer = model_summarizer.with_max_output_tokens(cap);
            }
            let summarizer = FaultTolerantCachingSummarizer::new(
                announced(Box::new(model_summarizer), compaction_notice),
                &policy,
            );
            ContextCompressionMiddleware::with_summarizer(
                policy,
                MemoryRecallSummarizer::wrap(Box::new(summarizer), memory_turn),
            )
        }
    }
}

/// What a compaction outside a turn folds against.
pub(crate) struct IdleFold<'a> {
    /// The caller's threshold: the conversation is at least this large.
    pub(crate) min_tokens: u64,
    /// The verbatim tail in tokens; `None` sizes it as a turn would against
    /// a `min_tokens` trigger. Either way the policy caps it at half of
    /// `min_tokens`, as it caps a turn's tail at half its trigger.
    pub(crate) keep_recent_tokens: Option<u64>,
    /// `[context] compaction_strategy`.
    pub(crate) strategy: crate::config::CompactionStrategy,
    /// The agent model's context window, when known.
    pub(crate) context_window: Option<u64>,
    /// The agent's model, for logs.
    pub(crate) model: &'a str,
}

/// `history` compacted with no model turn, or `None` when nothing was old
/// enough to fold.
///
/// This is a turn's compaction, driven over one request: the same
/// [`compression_middleware`] runs its `before_model` over `history` and its
/// `after_agent` reports the folded transcript, which becomes a
/// [`CompactionCarry`](tinyagents_harness::summarization::CompactionCarry)
/// re-applied to `history` exactly as the session driver applies a turn's.
/// The summary is placed after the system prompt (the default
/// `SummaryPlacement::User`). The caller has already decided the
/// conversation is at least `min_tokens`; the policy's trigger sits just
/// below that so the middleware compacts it.
///
/// # Errors
///
/// When the middleware fails (its failure policy normally recovers instead).
pub(crate) async fn compact_history(
    history: &[Message],
    fold: IdleFold<'_>,
    summarizer: CompactionSummarizer,
) -> tinyagents_harness::error::Result<Option<Vec<Message>>> {
    use tinyagents_harness::context::{RunConfig, RunContext};
    use tinyagents_harness::middleware::{AgentRun, Middleware};

    let trigger = fold.min_tokens.saturating_sub(1).max(1);
    let Some(policy) = compression_policy(fold.context_window, Some(trigger)) else {
        return Ok(None);
    };
    let mut mw = compression_middleware(
        policy,
        fold.strategy,
        fold.context_window,
        summarizer,
        fold.model,
        None,
        None,
    );
    if let Some(tokens) = fold.keep_recent_tokens {
        mw = mw.with_keep_recent_tokens(tokens);
    }
    let mut ctx = RunContext::new(RunConfig::new("compact_thread"), ());
    let mut request = tinyinference_llm::model::ModelRequest::new(history.to_vec());
    Middleware::<(), ()>::before_model(&mw, &mut ctx, &(), &mut request).await?;
    let mut run = AgentRun {
        messages: history.to_vec(),
        ..Default::default()
    };
    Middleware::<(), ()>::after_agent(&mw, &mut ctx, &(), &mut run).await?;
    let carry = run
        .compacted_history
        .as_deref()
        .and_then(tinyagents_harness::summarization::CompactionCarry::from_compacted_history);
    Ok(carry.map(|carry| carry.apply(history.to_vec())))
}

#[cfg(test)]
#[path = "context_wiring_tests.rs"]
mod tests;