Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

fix(media-proxy): require corroborating frames for nsfw (#2082)

Hampus03603662c641
diff --git a/fluxer_media_proxy/src/nsfw.rs b/fluxer_media_proxy/src/nsfw.rs
index 459b556b6..811877990 100644
--- a/fluxer_media_proxy/src/nsfw.rs
+++ b/fluxer_media_proxy/src/nsfw.rs
@@ -144,11 +144,11 @@ async fn check_buffers_inner(
return Err(Error::NsfwUnavailable);
}
let bytes = response.bytes().await.map_err(|_| Error::NsfwUnavailable)?;
- let probability = parse_batch_max_probability(&bytes)?;
- Ok(Result {
- probability,
- is_nsfw: probability >= cfg.threshold,
- })
+ let probabilities = parse_batch_probabilities(&bytes)?;
+ Ok(verdict_from_frame_probabilities(
+ &probabilities,
+ cfg.threshold,
+ ))
}
fn trim_trailing_slash(value: &str) -> &str {
@@ -168,24 +168,48 @@ pub fn parse_probability(body: &[u8]) -> std::result::Result<f32, Error> {
Err(Error::InvalidResponse)
}
-pub fn parse_batch_max_probability(body: &[u8]) -> std::result::Result<f32, Error> {
+pub const CORROBORATING_FRAMES_REQUIRED: usize = 2;
+
+pub fn parse_batch_probabilities(body: &[u8]) -> std::result::Result<Vec<f32>, Error> {
let value: Value = serde_json::from_slice(body).map_err(|_| Error::InvalidResponse)?;
let predictions = value
.get("predictions")
.and_then(Value::as_array)
.ok_or(Error::InvalidResponse)?;
- let mut max = 0.0f32;
+ let mut out = Vec::with_capacity(predictions.len());
for item in predictions {
let Some(object) = item.as_object() else {
continue;
};
+ let mut frame = None;
for key in ["nsfw_probability", "score", "probability", "nsfw"] {
if let Some(score) = object.get(key).and_then(number_as_f32) {
- max = max.max(score.clamp(0.0, 1.0));
+ let score = score.clamp(0.0, 1.0);
+ frame = Some(frame.map_or(score, |current: f32| current.max(score)));
}
}
+ if let Some(frame) = frame {
+ out.push(frame);
+ }
+ }
+ Ok(out)
+}
+
+pub fn verdict_from_frame_probabilities(probabilities: &[f32], threshold: f32) -> Result {
+ let max = probabilities.iter().copied().fold(0.0f32, f32::max);
+ let over = probabilities
+ .iter()
+ .filter(|probability| **probability >= threshold)
+ .count();
+ let required = if probabilities.len() >= CORROBORATING_FRAMES_REQUIRED {
+ CORROBORATING_FRAMES_REQUIRED
+ } else {
+ 1
+ };
+ Result {
+ probability: max,
+ is_nsfw: over >= required,
}
- Ok(max)
}
fn number_as_f32(value: &Value) -> Option<f32> {
@@ -197,21 +221,58 @@ mod tests {
use super::*;
#[test]
- fn parse_batch_max_probability_picks_highest_score() {
- assert!((parse_batch_max_probability(br#"{"predictions":[{"nsfw_probability":0.1},{"nsfw_probability":0.7},{"nsfw_probability":0.3}]}"#).unwrap() - 0.7).abs() < 0.001);
+ fn parse_batch_probabilities_reads_every_frame() {
+ let parsed = parse_batch_probabilities(
+ br#"{"predictions":[{"nsfw_probability":0.1},{"nsfw_probability":0.7},{"nsfw_probability":0.3}]}"#,
+ )
+ .unwrap();
+ assert_eq!(3, parsed.len());
+ assert!((parsed[1] - 0.7).abs() < 0.001);
assert!(
- (parse_batch_max_probability(br#"{"predictions":[]}"#).unwrap() - 0.0).abs() < 0.001
+ parse_batch_probabilities(br#"{"predictions":[]}"#)
+ .unwrap()
+ .is_empty()
);
assert_eq!(
Err(Error::InvalidResponse),
- parse_batch_max_probability(br#"{}"#)
+ parse_batch_probabilities(br#"{}"#)
);
assert_eq!(
Err(Error::InvalidResponse),
- parse_batch_max_probability(b"not-json")
+ parse_batch_probabilities(b"not-json")
);
}
+ #[test]
+ fn single_high_frame_does_not_flag_a_multi_frame_scan() {
+ let verdict = verdict_from_frame_probabilities(&[0.02, 0.99, 0.03], 0.95);
+ assert!(!verdict.is_nsfw);
+ assert!((verdict.probability - 0.99).abs() < 0.001);
+ }
+
+ #[test]
+ fn two_high_frames_flag_a_multi_frame_scan() {
+ assert!(verdict_from_frame_probabilities(&[0.02, 0.99, 0.97], 0.95).is_nsfw);
+ }
+
+ #[test]
+ fn a_lone_frame_still_flags_on_its_own() {
+ assert!(verdict_from_frame_probabilities(&[0.99], 0.95).is_nsfw);
+ assert!(!verdict_from_frame_probabilities(&[0.94], 0.95).is_nsfw);
+ }
+
+ #[test]
+ fn an_empty_scan_never_flags() {
+ let verdict = verdict_from_frame_probabilities(&[], 0.95);
+ assert!(!verdict.is_nsfw);
+ assert!((verdict.probability - 0.0).abs() < 0.001);
+ }
+
+ #[test]
+ fn frames_at_the_threshold_count_as_over() {
+ assert!(verdict_from_frame_probabilities(&[0.95, 0.95], 0.95).is_nsfw);
+ }
+
#[test]
fn disabled_when_endpoint_empty() {
let cfg = Config::disabled();