A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(unfurl): cache empty results briefly and key entries by provider (#1838)
Hampusc8ff177f85db
diff --git a/fluxer_unfurl/src/cache_policy.rs b/fluxer_unfurl/src/cache_policy.rsindex b0e61f238..d30c3d109 100644--- a/fluxer_unfurl/src/cache_policy.rs+++ b/fluxer_unfurl/src/cache_policy.rs@@ -1,3 +1,230 @@// SPDX-License-Identifier: AGPL-3.0-or-later-pub const FIXED_UNFURL_CACHE_TTL_SECS: u64 = 30 * 60;+use crate::types::{NsfwMode, UnfurlResponse, UnfurlResult};+use moka::Expiry;+use std::sync::Arc;+use std::time::{Duration, Instant};++pub const EMPTY_UNFURL_CACHE_TTL_SECS: u64 = 60;++const SCAN_TOKENS: [&str; 2] = ["scanned", "unscanned"];++const PROVIDER_CAPABILITIES: [&str; 4] = ["none", "youtube", "klipy", "youtube+klipy"];++pub fn nsfw_scan_token(nsfw_mode: NsfwMode) -> &'static str {+ match nsfw_mode {+ NsfwMode::Block | NsfwMode::Flag => SCAN_TOKENS[0],+ NsfwMode::Allow => SCAN_TOKENS[1],+ }+}++pub fn provider_capability(has_youtube_key: bool, has_klipy_key: bool) -> &'static str {+ match (has_youtube_key, has_klipy_key) {+ (true, true) => "youtube+klipy",+ (true, false) => "youtube",+ (false, true) => "klipy",+ (false, false) => "none",+ }+}++pub fn unfurl_cache_key(url: &str, nsfw_mode: NsfwMode, capability: &str) -> String {+ let scan = nsfw_scan_token(nsfw_mode);+ format!("{scan}:{capability}:{url}")+}++pub fn every_unfurl_cache_key(url: &str) -> Vec<String> {+ let mut keys = Vec::with_capacity(SCAN_TOKENS.len() * PROVIDER_CAPABILITIES.len());+ for scan in SCAN_TOKENS {+ for capability in PROVIDER_CAPABILITIES {+ keys.push(format!("{scan}:{capability}:{url}"));+ }+ }+ keys+}++pub fn default_empty_entry_lifetime() -> Duration {+ Duration::from_secs(EMPTY_UNFURL_CACHE_TTL_SECS)+}++pub struct UnfurlEntryExpiry {+ empty_entry_lifetime: Duration,+}++impl UnfurlEntryExpiry {+ pub fn new(empty_entry_lifetime: Duration) -> Self {+ Self {+ empty_entry_lifetime,+ }+ }++ fn lifetime(&self, has_embeds: bool) -> Option<Duration> {+ if has_embeds {+ None+ } else {+ Some(self.empty_entry_lifetime)+ }+ }+}++impl Expiry<String, Arc<UnfurlResult>> for UnfurlEntryExpiry {+ fn expire_after_create(+ &self,+ _key: &String,+ value: &Arc<UnfurlResult>,+ _created_at: Instant,+ ) -> Option<Duration> {+ self.lifetime(!value.embeds.is_empty())+ }++ fn expire_after_update(+ &self,+ _key: &String,+ value: &Arc<UnfurlResult>,+ _updated_at: Instant,+ _duration_until_expiry: Option<Duration>,+ ) -> Option<Duration> {+ self.lifetime(!value.embeds.is_empty())+ }+}++impl Expiry<String, UnfurlResponse> for UnfurlEntryExpiry {+ fn expire_after_create(+ &self,+ _key: &String,+ value: &UnfurlResponse,+ _created_at: Instant,+ ) -> Option<Duration> {+ self.lifetime(response_has_embeds(value))+ }++ fn expire_after_update(+ &self,+ _key: &String,+ value: &UnfurlResponse,+ _updated_at: Instant,+ _duration_until_expiry: Option<Duration>,+ ) -> Option<Duration> {+ self.lifetime(response_has_embeds(value))+ }+}++fn response_has_embeds(response: &UnfurlResponse) -> bool {+ match response {+ UnfurlResponse::Resolved(result) => !result.embeds.is_empty(),+ _ => false,+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::types::MessageEmbed;++ const URL: &str = "https://example.com/a";++ const EMPTY_UNFURL_CACHE_TTL_CEILING_SECS: u64 = 5 * 60;++ fn resolved(embed_count: usize) -> UnfurlResponse {+ UnfurlResponse::Resolved(Arc::new(UnfurlResult {+ embeds: (0..embed_count)+ .map(|_| MessageEmbed::new("link"))+ .collect(),+ cache_ttl_seconds: None,+ }))+ }++ #[test]+ fn every_key_covers_the_full_scan_and_capability_matrix() {+ let keys = every_unfurl_cache_key(URL);+ assert_eq!(keys.len(), 8);+ let unique: std::collections::HashSet<&String> = keys.iter().collect();+ assert_eq!(unique.len(), keys.len(), "invalidation keys must be unique");+ for nsfw_mode in [NsfwMode::Block, NsfwMode::Flag, NsfwMode::Allow] {+ for capability in PROVIDER_CAPABILITIES {+ assert!(+ keys.contains(&unfurl_cache_key(URL, nsfw_mode, capability)),+ "invalidation must cover {nsfw_mode:?}/{capability}"+ );+ }+ }+ }++ #[test]+ fn scanning_and_unscanned_requests_do_not_share_an_entry() {+ assert_ne!(+ unfurl_cache_key(URL, NsfwMode::Block, "none"),+ unfurl_cache_key(URL, NsfwMode::Allow, "none"),+ "an unscanned result carries no nsfw media flag and must not serve a scanning request"+ );+ }++ #[test]+ fn the_two_scanning_modes_share_a_single_entry() {+ assert_eq!(+ unfurl_cache_key(URL, NsfwMode::Block, "none"),+ unfurl_cache_key(URL, NsfwMode::Flag, "none")+ );+ }++ #[test]+ fn capability_distinguishes_every_key_combination() {+ let all = [+ provider_capability(false, false),+ provider_capability(true, false),+ provider_capability(false, true),+ provider_capability(true, true),+ ];+ let unique: std::collections::HashSet<&str> = all.iter().copied().collect();+ assert_eq!(unique.len(), all.len());+ for capability in all {+ assert_ne!(+ unfurl_cache_key(URL, NsfwMode::Block, capability),+ unfurl_cache_key(URL, NsfwMode::Block, "unrelated")+ );+ }+ }++ #[test]+ fn the_url_distinguishes_entries() {+ assert_ne!(+ unfurl_cache_key(URL, NsfwMode::Block, "none"),+ unfurl_cache_key("https://example.com/b", NsfwMode::Block, "none")+ );+ }++ #[test]+ fn the_empty_entry_lifetime_stays_a_short_retry_window() {+ assert!(+ default_empty_entry_lifetime()+ <= Duration::from_secs(EMPTY_UNFURL_CACHE_TTL_CEILING_SECS),+ "a site that was merely down must become retryable again quickly"+ );+ }++ #[test]+ fn the_expiry_reads_the_short_lifetime_off_an_empty_response() {+ let expiry = UnfurlEntryExpiry::new(default_empty_entry_lifetime());+ let now = Instant::now();+ let key = unfurl_cache_key(URL, NsfwMode::Block, "none");+ assert_eq!(+ Some(default_empty_entry_lifetime()),+ Expiry::<String, UnfurlResponse>::expire_after_create(&expiry, &key, &resolved(0), now)+ );+ assert_eq!(+ None,+ Expiry::<String, UnfurlResponse>::expire_after_create(&expiry, &key, &resolved(1), now)+ );+ assert_eq!(+ Some(default_empty_entry_lifetime()),+ Expiry::<String, Arc<UnfurlResult>>::expire_after_create(+ &expiry,+ &key,+ &Arc::new(UnfurlResult {+ embeds: Vec::new(),+ cache_ttl_seconds: None,+ }),+ now+ )+ );+ }+}diff --git a/fluxer_unfurl/src/main.rs b/fluxer_unfurl/src/main.rsindex af03caa29..84771ccf9 100644--- a/fluxer_unfurl/src/main.rs+++ b/fluxer_unfurl/src/main.rs@@ -40,11 +40,11 @@ async fn main() -> anyhow::Result<()> {match config.mode {Mode::Router => {- let router = UnfurlRouter::new(config.cache_max_entries);+ let router = UnfurlRouter::new(config.cache_max_entries, config.cache_ttl);fluxer_svc::router::run_router(&config, router, transport).await}Mode::Shard => {- let shard = UnfurlShard::new();+ let shard = UnfurlShard::new(config.cache_ttl);fluxer_svc::shard::run_shard(&config, shard, transport).await}}diff --git a/fluxer_unfurl/src/router_impl.rs b/fluxer_unfurl/src/router_impl.rsindex b61739717..629633855 100644--- a/fluxer_unfurl/src/router_impl.rs+++ b/fluxer_unfurl/src/router_impl.rs@@ -1,7 +1,10 @@// SPDX-License-Identifier: AGPL-3.0-or-later-use crate::cache_policy::FIXED_UNFURL_CACHE_TTL_SECS;-use crate::types::{NsfwMode, UnfurlRequest, UnfurlResponse};+use crate::cache_policy::{+ UnfurlEntryExpiry, default_empty_entry_lifetime, every_unfurl_cache_key, provider_capability,+ unfurl_cache_key,+};+use crate::types::{UnfurlRequest, UnfurlResponse};use fluxer_svc::router::RouterService;use moka::sync::Cache;use std::time::Duration;@@ -11,20 +14,27 @@ pub struct UnfurlRouter {}impl UnfurlRouter {- pub fn new(max_entries: u64) -> Self {+ pub fn new(max_entries: u64, lifetime: Duration) -> Self {+ Self::with_empty_entry_lifetime(max_entries, lifetime, default_empty_entry_lifetime())+ }++ fn with_empty_entry_lifetime(+ max_entries: u64,+ lifetime: Duration,+ empty_entry_lifetime: Duration,+ ) -> Self {Self {l1: Cache::builder().max_capacity(max_entries)- .time_to_live(Duration::from_secs(FIXED_UNFURL_CACHE_TTL_SECS))+ .time_to_live(lifetime)+ .expire_after(UnfurlEntryExpiry::new(empty_entry_lifetime)).build(),}}fn invalidate_all_variants(&self, url: &str) {- for mode in [NsfwMode::Block, NsfwMode::Flag, NsfwMode::Allow] {- for capability in PROVIDER_CAPABILITIES {- self.l1.invalidate(&unfurl_cache_key(url, mode, capability));- }+ for key in every_unfurl_cache_key(url) {+ self.l1.invalidate(&key);}}}@@ -63,7 +73,7 @@ impl RouterService for UnfurlRouter {unfurl_cache_key(url,nsfw_mode.unwrap_or_default(),- provider_capability(youtube_api_key, klipy_api_key)+ provider_capability(youtube_api_key.is_some(), klipy_api_key.is_some()))))}@@ -87,7 +97,7 @@ impl RouterService for UnfurlRouter {let key = unfurl_cache_key(url,nsfw_mode.unwrap_or_default(),- provider_capability(youtube_api_key, klipy_api_key),+ provider_capability(youtube_api_key.is_some(), klipy_api_key.is_some()),);self.l1.get(&key)}@@ -101,18 +111,19 @@ impl RouterService for UnfurlRouter {UnfurlRequest::Unfurl {url,nsfw_mode,+ cache_only,youtube_api_key,klipy_api_key,..} => {if let UnfurlResponse::Resolved(result) = resp- && !result.embeds.is_empty()+ && (!cache_only || !result.embeds.is_empty()){self.l1.insert(unfurl_cache_key(url,nsfw_mode.unwrap_or_default(),- provider_capability(youtube_api_key, klipy_api_key),+ provider_capability(youtube_api_key.is_some(), klipy_api_key.is_some()),),resp.clone(),);@@ -129,32 +140,14 @@ impl RouterService for UnfurlRouter {}}-const PROVIDER_CAPABILITIES: &[&str] = &["none", "youtube", "klipy", "youtube+klipy"];--fn provider_capability(- youtube_api_key: &Option<String>,- klipy_api_key: &Option<String>,-) -> &'static str {- match (youtube_api_key.is_some(), klipy_api_key.is_some()) {- (true, true) => "youtube+klipy",- (true, false) => "youtube",- (false, true) => "klipy",- (false, false) => "none",- }-}--fn unfurl_cache_key(url: &str, nsfw_mode: NsfwMode, capability: &str) -> String {- let mode = match nsfw_mode {- NsfwMode::Block => "block",- NsfwMode::Flag => "flag",- NsfwMode::Allow => "allow",- };- format!("{mode}:{capability}:{url}")-}-#[cfg(test)]mod tests {use super::*;+ use crate::types::NsfwMode;++ fn test_lifetime() -> Duration {+ Duration::from_secs(30 * 60)+ }fn request(cache_only: bool) -> UnfurlRequest {UnfurlRequest::Unfurl {@@ -174,6 +167,13 @@ mod tests {}))}+ fn empty_response() -> UnfurlResponse {+ UnfurlResponse::Resolved(std::sync::Arc::new(crate::types::UnfurlResult {+ embeds: Vec::new(),+ cache_ttl_seconds: None,+ }))+ }+fn unfurl(url: &str, youtube_api_key: Option<&str>) -> UnfurlRequest {UnfurlRequest::Unfurl {url: url.to_owned(),@@ -185,9 +185,42 @@ mod tests {}}+ fn unfurl_with_klipy(url: &str, klipy_api_key: Option<&str>) -> UnfurlRequest {+ UnfurlRequest::Unfurl {+ url: url.to_owned(),+ nsfw_mode: Some(NsfwMode::Block),+ bypass_cache: false,+ cache_only: false,+ youtube_api_key: None,+ klipy_api_key: klipy_api_key.map(str::to_owned),+ }+ }++ fn probe(url: &str) -> UnfurlRequest {+ UnfurlRequest::Unfurl {+ url: url.to_owned(),+ nsfw_mode: Some(NsfwMode::Block),+ bypass_cache: false,+ cache_only: true,+ youtube_api_key: None,+ klipy_api_key: None,+ }+ }++ fn unfurl_allowing_nsfw(url: &str) -> UnfurlRequest {+ UnfurlRequest::Unfurl {+ url: url.to_owned(),+ nsfw_mode: Some(NsfwMode::Allow),+ bypass_cache: false,+ cache_only: false,+ youtube_api_key: None,+ klipy_api_key: None,+ }+ }+#[test]fn configuring_a_youtube_key_does_not_reuse_the_keyless_cache_entry() {- let router = UnfurlRouter::new(16);+ let router = UnfurlRouter::new(16, test_lifetime());let url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ";router.l1_insert(&unfurl(url, None), &resolved_response());assert!(@@ -202,7 +235,7 @@ mod tests {#[test]fn invalidate_clears_every_provider_capability_variant() {- let router = UnfurlRouter::new(16);+ let router = UnfurlRouter::new(16, test_lifetime());let url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ";router.l1_insert(&unfurl(url, None), &resolved_response());router.l1_insert(&unfurl(url, Some("configured")), &resolved_response());@@ -223,4 +256,109 @@ mod tests {UnfurlRouter::coalesce_key(&request(false)));}+ #[test]+ fn a_url_that_produced_no_embed_is_remembered_instead_of_being_resolved_again() {+ let router = UnfurlRouter::new(16, test_lifetime());+ let url = "https://example.com/no-metadata";+ router.l1_insert(&unfurl(url, None), &empty_response());+ assert!(+ router.l1_lookup(&unfurl(url, None)).is_some(),+ "an empty resolution must be served from the upper cache, not re-resolved"+ );+ }++ #[test]+ fn an_empty_cache_only_probe_is_never_written_to_the_upper_cache() {+ let router = UnfurlRouter::new(16, test_lifetime());+ let url = "https://example.com/never-probed-into-cache";+ router.l1_insert(&probe(url), &empty_response());+ assert!(+ router.l1_lookup(&unfurl(url, None)).is_none(),+ "a cache-only miss means unknown, not resolved-to-nothing"+ );+ }++ #[test]+ fn a_cache_only_probe_that_found_embeds_still_warms_the_upper_cache() {+ let router = UnfurlRouter::new(16, test_lifetime());+ let url = "https://example.com/warm";+ router.l1_insert(&probe(url), &resolved_response());+ assert!(router.l1_lookup(&unfurl(url, None)).is_some());+ }++ #[test]+ fn configuring_a_klipy_key_does_not_reuse_the_keyless_cache_entry() {+ let router = UnfurlRouter::new(16, test_lifetime());+ let url = "https://klipy.com/gifs/example";+ router.l1_insert(&unfurl_with_klipy(url, None), &resolved_response());+ assert!(+ router+ .l1_lookup(&unfurl_with_klipy(url, Some("configured")))+ .is_none(),+ "a result resolved without a KLIPY key must not be served once a key is configured"+ );+ assert!(+ router.l1_lookup(&unfurl_with_klipy(url, None)).is_some(),+ "the keyless entry must still serve keyless requests"+ );+ }++ #[test]+ fn an_unscanned_entry_is_not_served_to_a_scanning_request() {+ let router = UnfurlRouter::new(16, test_lifetime());+ let url = "https://example.com/media";+ router.l1_insert(&unfurl_allowing_nsfw(url), &resolved_response());+ assert!(+ router.l1_lookup(&unfurl(url, None)).is_none(),+ "a result resolved with nsfw scanning off carries no nsfw flags and must not be reused"+ );+ assert!(router.l1_lookup(&unfurl_allowing_nsfw(url)).is_some());+ }++ #[test]+ fn invalidate_clears_both_scanning_and_unscanned_entries() {+ let router = UnfurlRouter::new(16, test_lifetime());+ let url = "https://example.com/both";+ router.l1_insert(&unfurl(url, None), &resolved_response());+ router.l1_insert(&unfurl_allowing_nsfw(url), &resolved_response());+ router.l1_insert(+ &UnfurlRequest::Invalidate {+ url: url.to_owned(),+ },+ &resolved_response(),+ );+ assert!(router.l1_lookup(&unfurl(url, None)).is_none());+ assert!(router.l1_lookup(&unfurl_allowing_nsfw(url)).is_none());+ }+ #[tokio::test]+ async fn an_empty_upper_cache_entry_lapses_long_before_a_populated_one() {+ let router =+ UnfurlRouter::with_empty_entry_lifetime(16, test_lifetime(), Duration::from_millis(50));+ let empty_url = "https://example.com/transient";+ let populated_url = "https://example.com/stable";+ router.l1_insert(&unfurl(empty_url, None), &empty_response());+ router.l1_insert(&unfurl(populated_url, None), &resolved_response());++ tokio::time::sleep(Duration::from_millis(200)).await;++ assert!(+ router.l1_lookup(&unfurl(empty_url, None)).is_none(),+ "a link that failed to unfurl must become retryable again quickly"+ );+ assert!(+ router.l1_lookup(&unfurl(populated_url, None)).is_some(),+ "a real embed must keep the full lifetime"+ );+ }+ #[tokio::test]+ async fn the_configured_lifetime_governs_the_upper_cache() {+ let router = UnfurlRouter::new(16, Duration::from_millis(80));+ let url = "https://example.com/configured";+ router.l1_insert(&unfurl(url, None), &resolved_response());+ tokio::time::sleep(Duration::from_millis(220)).await;+ assert!(+ router.l1_lookup(&unfurl(url, None)).is_none(),+ "the configured lifetime must govern the upper cache"+ );+ }}diff --git a/fluxer_unfurl/src/shard_impl.rs b/fluxer_unfurl/src/shard_impl.rsindex 2ed8bf838..8333f3aa0 100644--- a/fluxer_unfurl/src/shard_impl.rs+++ b/fluxer_unfurl/src/shard_impl.rs@@ -1,6 +1,9 @@// SPDX-License-Identifier: AGPL-3.0-or-later-use crate::cache_policy::FIXED_UNFURL_CACHE_TTL_SECS;+use crate::cache_policy::{+ UnfurlEntryExpiry, default_empty_entry_lifetime, every_unfurl_cache_key, provider_capability,+ unfurl_cache_key,+};use crate::embed_normalizer::normalize_embeds;use crate::media_proxy::MediaProxyClient;use crate::resolvers::{self, ResolveContext, ResolverResult};@@ -12,10 +15,10 @@ use std::time::Duration;use url::Url;const L2_MAX_ENTRIES: u64 = 50_000;-const L2_TTL: Duration = Duration::from_secs(FIXED_UNFURL_CACHE_TTL_SECS);pub struct UnfurlShard {cache: Cache<String, Arc<UnfurlResult>>,+ lifetime: Duration,http_client: reqwest::Client,resolvers: Vec<Box<dyn crate::resolvers::Resolver>>,media_proxy: MediaProxyClient,@@ -23,7 +26,7 @@ pub struct UnfurlShard {}impl UnfurlShard {- pub fn new() -> Self {+ pub fn new(lifetime: Duration) -> Self {let http_client = external_http_client();let media_proxy_http_client = internal_http_client();@@ -75,10 +78,8 @@ impl UnfurlShard {);Self {- cache: Cache::builder()- .max_capacity(L2_MAX_ENTRIES)- .time_to_live(L2_TTL)- .build(),+ cache: build_l2_cache(lifetime, default_empty_entry_lifetime()),+ lifetime,http_client,resolvers,media_proxy,@@ -86,6 +87,28 @@ impl UnfurlShard {}}+ #[cfg(test)]+ fn for_test() -> Self {+ Self::for_test_with_lifetimes(long_lifetime(), default_empty_entry_lifetime())+ }++ #[cfg(test)]+ fn for_test_with_lifetimes(lifetime: Duration, empty_entry_lifetime: Duration) -> Self {+ Self {+ cache: build_l2_cache(lifetime, empty_entry_lifetime),+ lifetime,+ http_client: external_http_client(),+ resolvers: resolvers::build_resolver_chain(),+ media_proxy: MediaProxyClient::new_with_public_endpoint(+ "http://media-proxy.invalid",+ "test-secret",+ None,+ internal_http_client(),+ ),+ static_cdn_endpoint: String::new(),+ }+ }+async fn resolve_url(&self,url_str: &str,@@ -163,11 +186,27 @@ impl UnfurlShard {fn finalize_result(&self, result: ResolverResult) -> UnfurlResult {UnfurlResult {embeds: normalize_embeds(result.embeds, &self.media_proxy),- cache_ttl_seconds: Some(FIXED_UNFURL_CACHE_TTL_SECS),+ cache_ttl_seconds: Some(self.lifetime.as_secs()),}}}+#[cfg(test)]+fn long_lifetime() -> Duration {+ Duration::from_secs(30 * 60)+}++fn build_l2_cache(+ lifetime: Duration,+ empty_entry_lifetime: Duration,+) -> Cache<String, Arc<UnfurlResult>> {+ Cache::builder()+ .max_capacity(L2_MAX_ENTRIES)+ .time_to_live(lifetime)+ .expire_after(UnfurlEntryExpiry::new(empty_entry_lifetime))+ .build()+}+fn base_http_client_builder() -> reqwest::ClientBuilder {reqwest::Client::builder().user_agent("Mozilla/5.0 (compatible; Fluxerbot/1.0; +https://fluxer.app)")@@ -209,7 +248,11 @@ impl ShardService for UnfurlShard {ref klipy_api_key,} => {let nsfw = nsfw_mode.unwrap_or_default();- let cache_key = unfurl_cache_key(url, nsfw);+ let cache_key = unfurl_cache_key(+ url,+ nsfw,+ provider_capability(youtube_api_key.is_some(), klipy_api_key.is_some()),+ );if !bypass_cache && let Some(cached) = self.cache.get(&cache_key).await {return Ok(UnfurlResponse::Resolved(cached));@@ -239,21 +282,13 @@ impl ShardService for UnfurlShard {}};- if !result.embeds.is_empty() {- self.cache.insert(cache_key, result.clone()).await;- }+ self.cache.insert(cache_key, result.clone()).await;Ok(UnfurlResponse::Resolved(result))}UnfurlRequest::Invalidate { ref url } => {- self.cache- .invalidate(&unfurl_cache_key(url, NsfwMode::Block))- .await;- self.cache- .invalidate(&unfurl_cache_key(url, NsfwMode::Flag))- .await;- self.cache- .invalidate(&unfurl_cache_key(url, NsfwMode::Allow))- .await;+ for key in every_unfurl_cache_key(url) {+ self.cache.invalidate(&key).await;+ }Ok(UnfurlResponse::Invalidated(InvalidatedResponse {invalidated: true,}))@@ -262,11 +297,230 @@ impl ShardService for UnfurlShard {}}-fn unfurl_cache_key(url: &str, nsfw_mode: NsfwMode) -> String {- let mode = match nsfw_mode {- NsfwMode::Block => "block",- NsfwMode::Flag => "flag",- NsfwMode::Allow => "allow",- };- format!("{mode}:{url}")+#[cfg(test)]+mod tests {+ use super::*;+ use crate::types::MessageEmbed;++ const BLOCKED_URL: &str = "http://127.0.0.1/unreachable";++ fn unfurl(url: &str, youtube_api_key: Option<&str>) -> UnfurlRequest {+ UnfurlRequest::Unfurl {+ url: url.to_owned(),+ nsfw_mode: Some(NsfwMode::Block),+ bypass_cache: false,+ cache_only: false,+ youtube_api_key: youtube_api_key.map(str::to_owned),+ klipy_api_key: None,+ }+ }++ fn probe(url: &str) -> UnfurlRequest {+ UnfurlRequest::Unfurl {+ url: url.to_owned(),+ nsfw_mode: Some(NsfwMode::Block),+ bypass_cache: false,+ cache_only: true,+ youtube_api_key: None,+ klipy_api_key: None,+ }+ }++ fn keyless_block_key(url: &str) -> String {+ unfurl_cache_key(url, NsfwMode::Block, provider_capability(false, false))+ }++ fn embed_count(response: &UnfurlResponse) -> usize {+ match response {+ UnfurlResponse::Resolved(result) => result.embeds.len(),+ _ => panic!("expected a resolved response"),+ }+ }++ async fn seed_keyless_entry(shard: &UnfurlShard) {+ shard+ .cache+ .insert(+ unfurl_cache_key(+ BLOCKED_URL,+ NsfwMode::Block,+ provider_capability(false, false),+ ),+ Arc::new(UnfurlResult {+ embeds: vec![MessageEmbed::new("link")],+ cache_ttl_seconds: Some(long_lifetime().as_secs()),+ }),+ )+ .await;+ }++ #[tokio::test]+ async fn a_shard_entry_resolved_without_a_youtube_key_is_not_served_once_one_is_configured() {+ let shard = UnfurlShard::for_test();+ seed_keyless_entry(&shard).await;++ assert_eq!(+ 1,+ embed_count(&shard.handle(unfurl(BLOCKED_URL, None)).await.unwrap()),+ "the keyless entry must still serve keyless requests"+ );+ assert_eq!(+ 0,+ embed_count(+ &shard+ .handle(unfurl(BLOCKED_URL, Some("configured")))+ .await+ .unwrap()+ ),+ "a result resolved without a YouTube key must not be served once a key is configured"+ );+ }++ #[tokio::test]+ async fn invalidate_clears_the_shard_entry_for_every_provider_capability() {+ let shard = UnfurlShard::for_test();+ seed_keyless_entry(&shard).await;++ shard+ .handle(UnfurlRequest::Invalidate {+ url: BLOCKED_URL.to_owned(),+ })+ .await+ .unwrap();++ assert_eq!(+ 0,+ embed_count(&shard.handle(unfurl(BLOCKED_URL, None)).await.unwrap())+ );+ }+ #[tokio::test]+ async fn a_url_that_produced_no_embed_is_written_to_the_shard_cache() {+ let shard = UnfurlShard::for_test();++ assert_eq!(+ 0,+ embed_count(&shard.handle(unfurl(BLOCKED_URL, None)).await.unwrap())+ );+ assert!(+ shard+ .cache+ .get(&keyless_block_key(BLOCKED_URL))+ .await+ .is_some(),+ "an empty resolution must be remembered so the next message does not refetch the page"+ );+ }++ #[tokio::test]+ async fn a_cache_only_probe_never_writes_an_entry() {+ let shard = UnfurlShard::for_test();++ assert_eq!(+ 0,+ embed_count(&shard.handle(probe(BLOCKED_URL)).await.unwrap())+ );+ assert!(+ shard+ .cache+ .get(&keyless_block_key(BLOCKED_URL))+ .await+ .is_none(),+ "a cache-only miss means unknown, not resolved-to-nothing"+ );+ }++ #[tokio::test]+ async fn an_unscanned_entry_is_not_served_to_a_scanning_request() {+ let shard = UnfurlShard::for_test();+ shard+ .cache+ .insert(+ unfurl_cache_key(+ BLOCKED_URL,+ NsfwMode::Allow,+ provider_capability(false, false),+ ),+ Arc::new(UnfurlResult {+ embeds: vec![MessageEmbed::new("link")],+ cache_ttl_seconds: Some(long_lifetime().as_secs()),+ }),+ )+ .await;++ assert_eq!(+ 0,+ embed_count(&shard.handle(unfurl(BLOCKED_URL, None)).await.unwrap()),+ "a result resolved with nsfw scanning off must not be served to a scanning request"+ );+ }+ #[tokio::test]+ async fn an_empty_shard_entry_lapses_long_before_a_populated_one() {+ let shard =+ UnfurlShard::for_test_with_lifetimes(long_lifetime(), Duration::from_millis(50));+ let empty_key = keyless_block_key("https://example.com/transient");+ let populated_key = keyless_block_key("https://example.com/stable");+ shard+ .cache+ .insert(+ empty_key.clone(),+ Arc::new(UnfurlResult {+ embeds: Vec::new(),+ cache_ttl_seconds: None,+ }),+ )+ .await;+ shard+ .cache+ .insert(+ populated_key.clone(),+ Arc::new(UnfurlResult {+ embeds: vec![MessageEmbed::new("link")],+ cache_ttl_seconds: Some(long_lifetime().as_secs()),+ }),+ )+ .await;++ tokio::time::sleep(Duration::from_millis(200)).await;++ assert!(+ shard.cache.get(&empty_key).await.is_none(),+ "a link that failed to unfurl must become retryable again quickly"+ );+ assert!(+ shard.cache.get(&populated_key).await.is_some(),+ "a real embed must keep the full lifetime"+ );+ }+ #[tokio::test]+ async fn the_configured_lifetime_governs_the_shard_cache_and_the_reported_ttl() {+ let shard = UnfurlShard::for_test_with_lifetimes(+ Duration::from_millis(80),+ default_empty_entry_lifetime(),+ );+ let finalized = shard.finalize_result(crate::resolvers::ResolverResult {+ embeds: vec![MessageEmbed::new("link")],+ });+ assert_eq!(+ Some(Duration::from_millis(80).as_secs()),+ finalized.cache_ttl_seconds,+ "the reported ttl must be the configured lifetime, not a compiled-in constant"+ );++ let key = keyless_block_key("https://example.com/configured");+ shard+ .cache+ .insert(+ key.clone(),+ Arc::new(UnfurlResult {+ embeds: vec![MessageEmbed::new("link")],+ cache_ttl_seconds: None,+ }),+ )+ .await;+ tokio::time::sleep(Duration::from_millis(220)).await;+ assert!(+ shard.cache.get(&key).await.is_none(),+ "the configured lifetime must govern the shard cache"+ );+ }}