Skip to content
Python
11.2 KB
Raw Blame
use super::*;
use anyhow::bail;
use futures_util::FutureExt;
use std::panic::AssertUnwindSafe;
use std::time::{Duration, Instant};
async fn bounded<T>(
    ms: u64,
    cancel: &CancellationToken,
    future: impl Future<Output = Result<T>>,
) -> Result<T> {
    let future = AssertUnwindSafe(future).catch_unwind();
    tokio::select! {biased;_=cancel.cancelled()=>bail!("Safety evaluation cancelled"),r=tokio::time::timeout(Duration::from_millis(ms),future)=>r.map_err(|_|anyhow::anyhow!("Safety evaluation timed out"))?.map_err(|_|anyhow::anyhow!("Safety handler panicked"))?}
}
async fn detect(
    rule: &Rule,
    context: &Context,
    models: Option<&dyn ModelDetector>,
    cancel: &CancellationToken,
) -> Result<Vec<Match>> {
    match &rule.detector {
        Detector::Model {
            model,
            instructions,
        } => {
            let driver = models.ok_or_else(|| anyhow::anyhow!("Guard model unavailable"))?;
            let found = bounded(rule.timeout_ms, cancel, async {
                driver
                    .detect(
                        model.clone(),
                        instructions.clone(),
                        context.clone(),
                        cancel.clone(),
                    )
                    .await
            })
            .await?;
            fragments(&context.text, &found)
        }
        detector => detectors::detect(detector, &context.text),
    }
}
fn finding(
    c: &Context,
    name: &str,
    version: &str,
    outcome: &str,
    code: Option<String>,
    count: usize,
    started: Instant,
) -> Finding {
    Finding {
        call_id: c.call_id.clone(),
        tool: c.tool.clone(),
        evaluation_id: c.evaluation_id.clone(),
        phase: c.phase,
        check: name.into(),
        version: version.into(),
        outcome: outcome.into(),
        code,
        match_count: count,
        duration_ms: started.elapsed().as_millis().min(u64::MAX as u128) as u64,
    }
}
fn safe_block(decision: Decision, rules: &[&Rule], context: &Context) -> Decision {
    let Decision::Block {
        mut code,
        mut feedback_to_model,
    } = decision
    else {
        return decision;
    };
    // Application feedback is a new output surface. Never echo a detected
    // fragment into history, public errors or compliance reason codes.
    let unsafe_text = |text: &str| {
        context
            .matches
            .iter()
            .any(|m| !m.text.is_empty() && text.contains(&m.text))
            || rules.iter().any(|r| match &r.detector {
                Detector::Model { .. } => false,
                detector => detectors::detect(detector, text).map_or(true, |m| !m.is_empty()),
            })
    };
    if unsafe_text(&code) {
        code = "CONTENT_BLOCKED".into();
    }
    if feedback_to_model.trim().is_empty()
        || unsafe_text(&feedback_to_model)
        || rules
            .iter()
            .any(|r| matches!(r.detector, Detector::Model { .. }))
    {
        if let Decision::Block {
            feedback_to_model: safe,
            ..
        } = Decision::block("CONTENT_BLOCKED")
        {
            feedback_to_model = safe;
        }
    }
    Decision::Block {
        code,
        feedback_to_model,
    }
}
fn reject(out: &mut Evaluation, decision: Decision) {
    out.block = Some(decision);
}
pub async fn apply(
    config: &Config,
    registry: &Registry,
    models: Option<&dyn ModelDetector>,
    mut context: Context,
    cancel: CancellationToken,
) -> Result<Evaluation> {
    config.validate()?;
    ensure!(
        context.text.len() <= MAX_TEXT,
        "Text exceeds safety size limit"
    );
    let mut out = Evaluation {
        text: context.text.clone(),
        findings: vec![],
        block: None,
        unavailable: false,
    };
    let rules = config.rules(context.phase, context.tool.as_deref());
    let mut transformed = false;
    for rule in &rules {
        let started = Instant::now();
        context.text = out.text.clone();
        context.matches.clear();
        let matches = match detect(rule, &context, models, &cancel).await {
            Ok(m) => m,
            Err(_) => {
                out.findings
                    .push(unavailable(&context, &rule.name, &rule.version));
                out.unavailable = true;
                return Ok(out);
            }
        };
        let count = matches.len();
        if matches.is_empty() {
            out.findings.push(finding(
                &context,
                &rule.name,
                &rule.version,
                "allow",
                None,
                0,
                started,
            ));
            continue;
        }
        context.matches = matches;
        let decision = match &rule.action {
            Action::Block => Decision::block("CONTENT_BLOCKED"),
            Action::Redact {
                replacement,
                keep_last,
            } => Decision::Replace {
                text: detectors::redact(
                    &context.text,
                    &context.matches,
                    replacement,
                    *keep_last,
                    matches!(&rule.detector, Detector::CreditCard),
                ),
            },
            Action::Callback { handler } => match registry.handler(handler) {
                Ok(h) => match bounded(rule.timeout_ms, &cancel, async {
                    h.call(context.clone(), cancel.clone()).await
                })
                .await
                {
                    Ok(d) if d.validate().is_ok() && !matches!(&d, Decision::Allow) => d,
                    _ => {
                        out.findings
                            .push(unavailable(&context, &rule.name, &rule.version));
                        out.unavailable = true;
                        return Ok(out);
                    }
                },
                Err(_) => {
                    out.findings
                        .push(unavailable(&context, &rule.name, &rule.version));
                    out.unavailable = true;
                    return Ok(out);
                }
            },
        };
        match decision {
            Decision::Allow => out.findings.push(finding(
                &context,
                &rule.name,
                &rule.version,
                "allow",
                None,
                count,
                started,
            )),
            Decision::Replace { text } => {
                ensure!(text.len() <= MAX_TEXT, "Replacement exceeds safety limit");
                out.text = text;
                transformed = true;
                out.findings.push(finding(
                    &context,
                    &rule.name,
                    &rule.version,
                    "redacted",
                    None,
                    count,
                    started,
                ));
            }
            decision @ Decision::Block { .. } => {
                let decision = safe_block(decision, &rules, &context);
                let code = if let Decision::Block { code, .. } = &decision {
                    Some(code.clone())
                } else {
                    None
                };
                out.findings.push(finding(
                    &context,
                    &rule.name,
                    &rule.version,
                    "blocked",
                    code,
                    count,
                    started,
                ));
                reject(&mut out, decision);
                return Ok(out);
            }
        }
    }
    // A later custom replacement must never reintroduce an earlier prohibited
    // value. Revalidate without applying transforms/callback effects again.
    if transformed {
        context.text = out.text.clone();
        context.matches.clear();
        for rule in &rules {
            let started = Instant::now();
            let matches = match detect(rule, &context, models, &cancel).await {
                Ok(m) => m,
                Err(_) => {
                    out.findings
                        .push(unavailable(&context, &rule.name, &rule.version));
                    out.unavailable = true;
                    return Ok(out);
                }
            };
            if !matches.is_empty() {
                out.findings.push(finding(
                    &context,
                    &rule.name,
                    &rule.version,
                    "blocked",
                    Some("SANITIZATION_INCOMPLETE".into()),
                    matches.len(),
                    started,
                ));
                reject(&mut out, Decision::block("SANITIZATION_INCOMPLETE"));
                return Ok(out);
            }
        }
    }
    context.text = out.text.clone();
    context.matches.clear();
    if context.phase == Phase::ToolCall && context.arguments.is_some() {
        context.arguments = Some(
            serde_json::from_str(&context.text)
                .map_err(|_| anyhow::anyhow!("Sanitized tool arguments are not valid JSON"))?,
        );
    }
    for hook in config.hooks.iter().filter(|h| {
        h.when != HookPhase::OnEvent
            && h.when.applies(context.phase)
            && (h.tools.is_empty() || context.tool.as_ref().is_some_and(|t| h.tools.contains(t)))
    }) {
        let started = Instant::now();
        let result = match registry.handler(&hook.handler) {
            Ok(h) => {
                bounded(hook.timeout_ms, &cancel, async {
                    h.call(context.clone(), cancel.clone()).await
                })
                .await
            }
            Err(e) => Err(e),
        };
        match result {
            Ok(Decision::Allow) => out.findings.push(finding(
                &context,
                &hook.name,
                &hook.version,
                "allow",
                None,
                0,
                started,
            )),
            Ok(decision @ Decision::Block { .. }) if decision.validate().is_ok() => {
                let decision = safe_block(decision, &rules, &context);
                let code = if let Decision::Block { code, .. } = &decision {
                    Some(code.clone())
                } else {
                    None
                };
                out.findings.push(finding(
                    &context,
                    &hook.name,
                    &hook.version,
                    "blocked",
                    code,
                    0,
                    started,
                ));
                reject(&mut out, decision);
                return Ok(out);
            }
            _ => {
                out.findings
                    .push(unavailable(&context, &hook.name, &hook.version));
                out.unavailable = true;
                return Ok(out);
            }
        }
    }
    Ok(out)
}
/// Observer failure is diagnostic, never authority. Caller queues/persists this
/// outside the action path; the action's original outcome remains unchanged.
pub async fn observe(
    hook: &Hook,
    registry: &Registry,
    context: Context,
    cancel: CancellationToken,
) -> bool {
    if hook.when != HookPhase::OnEvent {
        return false;
    }
    match registry.handler(&hook.handler) {
        Ok(h) => bounded(hook.timeout_ms, &cancel, async {
            h.call(context, cancel.clone()).await
        })
        .await
        .is_ok(),
        Err(_) => false,
    }
}