Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

fix(unfurl): cache empty results briefly and key entries by provider (#1838)

Hampusc8ff177f85db
diff --git a/fluxer_unfurl/src/cache_policy.rs b/fluxer_unfurl/src/cache_policy.rs
index b0e61f238..d30c3d109 100644
--- a/fluxer_unfurl/src/cache_policy.rs
+++ b/fluxer_unfurl/src/cache_policy.rs
@@ -1,3 +1,230 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
-pub const FIXED_UNFURL_CACHE_TTL_SECS: u64 = 30 * 60;
+use crate::types::{NsfwMode, UnfurlResponse, UnfurlResult};
+use moka::Expiry;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+pub const EMPTY_UNFURL_CACHE_TTL_SECS: u64 = 60;
+
+const SCAN_TOKENS: [&str; 2] = ["scanned", "unscanned"];
+
+const PROVIDER_CAPABILITIES: [&str; 4] = ["none", "youtube", "klipy", "youtube+klipy"];
+
+pub fn nsfw_scan_token(nsfw_mode: NsfwMode) -> &'static str {
+ match nsfw_mode {
+ NsfwMode::Block | NsfwMode::Flag => SCAN_TOKENS[0],
+ NsfwMode::Allow => SCAN_TOKENS[1],
+ }
+}
+
+pub fn provider_capability(has_youtube_key: bool, has_klipy_key: bool) -> &'static str {
+ match (has_youtube_key, has_klipy_key) {
+ (true, true) => "youtube+klipy",
+ (true, false) => "youtube",
+ (false, true) => "klipy",
+ (false, false) => "none",
+ }
+}
+
+pub fn unfurl_cache_key(url: &str, nsfw_mode: NsfwMode, capability: &str) -> String {
+ let scan = nsfw_scan_token(nsfw_mode);
+ format!("{scan}:{capability}:{url}")
+}
+
+pub fn every_unfurl_cache_key(url: &str) -> Vec<String> {
+ let mut keys = Vec::with_capacity(SCAN_TOKENS.len() * PROVIDER_CAPABILITIES.len());
+ for scan in SCAN_TOKENS {
+ for capability in PROVIDER_CAPABILITIES {
+ keys.push(format!("{scan}:{capability}:{url}"));
+ }
+ }
+ keys
+}
+
+pub fn default_empty_entry_lifetime() -> Duration {
+ Duration::from_secs(EMPTY_UNFURL_CACHE_TTL_SECS)
+}
+
+pub struct UnfurlEntryExpiry {
+ empty_entry_lifetime: Duration,
+}
+
+impl UnfurlEntryExpiry {
+ pub fn new(empty_entry_lifetime: Duration) -> Self {
+ Self {
+ empty_entry_lifetime,
+ }
+ }
+
+ fn lifetime(&self, has_embeds: bool) -> Option<Duration> {
+ if has_embeds {
+ None
+ } else {
+ Some(self.empty_entry_lifetime)
+ }
+ }
+}
+
+impl Expiry<String, Arc<UnfurlResult>> for UnfurlEntryExpiry {
+ fn expire_after_create(
+ &self,
+ _key: &String,
+ value: &Arc<UnfurlResult>,
+ _created_at: Instant,
+ ) -> Option<Duration> {
+ self.lifetime(!value.embeds.is_empty())
+ }
+
+ fn expire_after_update(
+ &self,
+ _key: &String,
+ value: &Arc<UnfurlResult>,
+ _updated_at: Instant,
+ _duration_until_expiry: Option<Duration>,
+ ) -> Option<Duration> {
+ self.lifetime(!value.embeds.is_empty())
+ }
+}
+
+impl Expiry<String, UnfurlResponse> for UnfurlEntryExpiry {
+ fn expire_after_create(
+ &self,
+ _key: &String,
+ value: &UnfurlResponse,
+ _created_at: Instant,
+ ) -> Option<Duration> {
+ self.lifetime(response_has_embeds(value))
+ }
+
+ fn expire_after_update(
+ &self,
+ _key: &String,
+ value: &UnfurlResponse,
+ _updated_at: Instant,
+ _duration_until_expiry: Option<Duration>,
+ ) -> Option<Duration> {
+ self.lifetime(response_has_embeds(value))
+ }
+}
+
+fn response_has_embeds(response: &UnfurlResponse) -> bool {
+ match response {
+ UnfurlResponse::Resolved(result) => !result.embeds.is_empty(),
+ _ => false,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::types::MessageEmbed;
+
+ const URL: &str = "https://example.com/a";
+
+ const EMPTY_UNFURL_CACHE_TTL_CEILING_SECS: u64 = 5 * 60;
+
+ fn resolved(embed_count: usize) -> UnfurlResponse {
+ UnfurlResponse::Resolved(Arc::new(UnfurlResult {
+ embeds: (0..embed_count)
+ .map(|_| MessageEmbed::new("link"))
+ .collect(),
+ cache_ttl_seconds: None,
+ }))
+ }
+
+ #[test]
+ fn every_key_covers_the_full_scan_and_capability_matrix() {
+ let keys = every_unfurl_cache_key(URL);
+ assert_eq!(keys.len(), 8);
+ let unique: std::collections::HashSet<&String> = keys.iter().collect();
+ assert_eq!(unique.len(), keys.len(), "invalidation keys must be unique");
+ for nsfw_mode in [NsfwMode::Block, NsfwMode::Flag, NsfwMode::Allow] {
+ for capability in PROVIDER_CAPABILITIES {
+ assert!(
+ keys.contains(&unfurl_cache_key(URL, nsfw_mode, capability)),
+ "invalidation must cover {nsfw_mode:?}/{capability}"
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn scanning_and_unscanned_requests_do_not_share_an_entry() {
+ assert_ne!(
+ unfurl_cache_key(URL, NsfwMode::Block, "none"),
+ unfurl_cache_key(URL, NsfwMode::Allow, "none"),
+ "an unscanned result carries no nsfw media flag and must not serve a scanning request"
+ );
+ }
+
+ #[test]
+ fn the_two_scanning_modes_share_a_single_entry() {
+ assert_eq!(
+ unfurl_cache_key(URL, NsfwMode::Block, "none"),
+ unfurl_cache_key(URL, NsfwMode::Flag, "none")
+ );
+ }
+
+ #[test]
+ fn capability_distinguishes_every_key_combination() {
+ let all = [
+ provider_capability(false, false),
+ provider_capability(true, false),
+ provider_capability(false, true),
+ provider_capability(true, true),
+ ];
+ let unique: std::collections::HashSet<&str> = all.iter().copied().collect();
+ assert_eq!(unique.len(), all.len());
+ for capability in all {
+ assert_ne!(
+ unfurl_cache_key(URL, NsfwMode::Block, capability),
+ unfurl_cache_key(URL, NsfwMode::Block, "unrelated")
+ );
+ }
+ }
+
+ #[test]
+ fn the_url_distinguishes_entries() {
+ assert_ne!(
+ unfurl_cache_key(URL, NsfwMode::Block, "none"),
+ unfurl_cache_key("https://example.com/b", NsfwMode::Block, "none")
+ );
+ }
+
+ #[test]
+ fn the_empty_entry_lifetime_stays_a_short_retry_window() {
+ assert!(
+ default_empty_entry_lifetime()
+ <= Duration::from_secs(EMPTY_UNFURL_CACHE_TTL_CEILING_SECS),
+ "a site that was merely down must become retryable again quickly"
+ );
+ }
+
+ #[test]
+ fn the_expiry_reads_the_short_lifetime_off_an_empty_response() {
+ let expiry = UnfurlEntryExpiry::new(default_empty_entry_lifetime());
+ let now = Instant::now();
+ let key = unfurl_cache_key(URL, NsfwMode::Block, "none");
+ assert_eq!(
+ Some(default_empty_entry_lifetime()),
+ Expiry::<String, UnfurlResponse>::expire_after_create(&expiry, &key, &resolved(0), now)
+ );
+ assert_eq!(
+ None,
+ Expiry::<String, UnfurlResponse>::expire_after_create(&expiry, &key, &resolved(1), now)
+ );
+ assert_eq!(
+ Some(default_empty_entry_lifetime()),
+ Expiry::<String, Arc<UnfurlResult>>::expire_after_create(
+ &expiry,
+ &key,
+ &Arc::new(UnfurlResult {
+ embeds: Vec::new(),
+ cache_ttl_seconds: None,
+ }),
+ now
+ )
+ );
+ }
+}
diff --git a/fluxer_unfurl/src/main.rs b/fluxer_unfurl/src/main.rs
index af03caa29..84771ccf9 100644
--- a/fluxer_unfurl/src/main.rs
+++ b/fluxer_unfurl/src/main.rs
@@ -40,11 +40,11 @@ async fn main() -> anyhow::Result<()> {
match config.mode {
Mode::Router => {
- let router = UnfurlRouter::new(config.cache_max_entries);
+ let router = UnfurlRouter::new(config.cache_max_entries, config.cache_ttl);
fluxer_svc::router::run_router(&config, router, transport).await
}
Mode::Shard => {
- let shard = UnfurlShard::new();
+ let shard = UnfurlShard::new(config.cache_ttl);
fluxer_svc::shard::run_shard(&config, shard, transport).await
}
}
diff --git a/fluxer_unfurl/src/router_impl.rs b/fluxer_unfurl/src/router_impl.rs
index b61739717..629633855 100644
--- a/fluxer_unfurl/src/router_impl.rs
+++ b/fluxer_unfurl/src/router_impl.rs
@@ -1,7 +1,10 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
-use crate::cache_policy::FIXED_UNFURL_CACHE_TTL_SECS;
-use crate::types::{NsfwMode, UnfurlRequest, UnfurlResponse};
+use crate::cache_policy::{
+ UnfurlEntryExpiry, default_empty_entry_lifetime, every_unfurl_cache_key, provider_capability,
+ unfurl_cache_key,
+};
+use crate::types::{UnfurlRequest, UnfurlResponse};
use fluxer_svc::router::RouterService;
use moka::sync::Cache;
use std::time::Duration;
@@ -11,20 +14,27 @@ pub struct UnfurlRouter {
}
impl UnfurlRouter {
- pub fn new(max_entries: u64) -> Self {
+ pub fn new(max_entries: u64, lifetime: Duration) -> Self {
+ Self::with_empty_entry_lifetime(max_entries, lifetime, default_empty_entry_lifetime())
+ }
+
+ fn with_empty_entry_lifetime(
+ max_entries: u64,
+ lifetime: Duration,
+ empty_entry_lifetime: Duration,
+ ) -> Self {
Self {
l1: Cache::builder()
.max_capacity(max_entries)
- .time_to_live(Duration::from_secs(FIXED_UNFURL_CACHE_TTL_SECS))
+ .time_to_live(lifetime)
+ .expire_after(UnfurlEntryExpiry::new(empty_entry_lifetime))
.build(),
}
}
fn invalidate_all_variants(&self, url: &str) {
- for mode in [NsfwMode::Block, NsfwMode::Flag, NsfwMode::Allow] {
- for capability in PROVIDER_CAPABILITIES {
- self.l1.invalidate(&unfurl_cache_key(url, mode, capability));
- }
+ for key in every_unfurl_cache_key(url) {
+ self.l1.invalidate(&key);
}
}
}
@@ -63,7 +73,7 @@ impl RouterService for UnfurlRouter {
unfurl_cache_key(
url,
nsfw_mode.unwrap_or_default(),
- provider_capability(youtube_api_key, klipy_api_key)
+ provider_capability(youtube_api_key.is_some(), klipy_api_key.is_some())
)
))
}
@@ -87,7 +97,7 @@ impl RouterService for UnfurlRouter {
let key = unfurl_cache_key(
url,
nsfw_mode.unwrap_or_default(),
- provider_capability(youtube_api_key, klipy_api_key),
+ provider_capability(youtube_api_key.is_some(), klipy_api_key.is_some()),
);
self.l1.get(&key)
}
@@ -101,18 +111,19 @@ impl RouterService for UnfurlRouter {
UnfurlRequest::Unfurl {
url,
nsfw_mode,
+ cache_only,
youtube_api_key,
klipy_api_key,
..
} => {
if let UnfurlResponse::Resolved(result) = resp
- && !result.embeds.is_empty()
+ && (!cache_only || !result.embeds.is_empty())
{
self.l1.insert(
unfurl_cache_key(
url,
nsfw_mode.unwrap_or_default(),
- provider_capability(youtube_api_key, klipy_api_key),
+ provider_capability(youtube_api_key.is_some(), klipy_api_key.is_some()),
),
resp.clone(),
);
@@ -129,32 +140,14 @@ impl RouterService for UnfurlRouter {
}
}
-const PROVIDER_CAPABILITIES: &[&str] = &["none", "youtube", "klipy", "youtube+klipy"];
-
-fn provider_capability(
- youtube_api_key: &Option<String>,
- klipy_api_key: &Option<String>,
-) -> &'static str {
- match (youtube_api_key.is_some(), klipy_api_key.is_some()) {
- (true, true) => "youtube+klipy",
- (true, false) => "youtube",
- (false, true) => "klipy",
- (false, false) => "none",
- }
-}
-
-fn unfurl_cache_key(url: &str, nsfw_mode: NsfwMode, capability: &str) -> String {
- let mode = match nsfw_mode {
- NsfwMode::Block => "block",
- NsfwMode::Flag => "flag",
- NsfwMode::Allow => "allow",
- };
- format!("{mode}:{capability}:{url}")
-}
-
#[cfg(test)]
mod tests {
use super::*;
+ use crate::types::NsfwMode;
+
+ fn test_lifetime() -> Duration {
+ Duration::from_secs(30 * 60)
+ }
fn request(cache_only: bool) -> UnfurlRequest {
UnfurlRequest::Unfurl {
@@ -174,6 +167,13 @@ mod tests {
}))
}
+ fn empty_response() -> UnfurlResponse {
+ UnfurlResponse::Resolved(std::sync::Arc::new(crate::types::UnfurlResult {
+ embeds: Vec::new(),
+ cache_ttl_seconds: None,
+ }))
+ }
+
fn unfurl(url: &str, youtube_api_key: Option<&str>) -> UnfurlRequest {
UnfurlRequest::Unfurl {
url: url.to_owned(),
@@ -185,9 +185,42 @@ mod tests {
}
}
+ fn unfurl_with_klipy(url: &str, klipy_api_key: Option<&str>) -> UnfurlRequest {
+ UnfurlRequest::Unfurl {
+ url: url.to_owned(),
+ nsfw_mode: Some(NsfwMode::Block),
+ bypass_cache: false,
+ cache_only: false,
+ youtube_api_key: None,
+ klipy_api_key: klipy_api_key.map(str::to_owned),
+ }
+ }
+
+ fn probe(url: &str) -> UnfurlRequest {
+ UnfurlRequest::Unfurl {
+ url: url.to_owned(),
+ nsfw_mode: Some(NsfwMode::Block),
+ bypass_cache: false,
+ cache_only: true,
+ youtube_api_key: None,
+ klipy_api_key: None,
+ }
+ }
+
+ fn unfurl_allowing_nsfw(url: &str) -> UnfurlRequest {
+ UnfurlRequest::Unfurl {
+ url: url.to_owned(),
+ nsfw_mode: Some(NsfwMode::Allow),
+ bypass_cache: false,
+ cache_only: false,
+ youtube_api_key: None,
+ klipy_api_key: None,
+ }
+ }
+
#[test]
fn configuring_a_youtube_key_does_not_reuse_the_keyless_cache_entry() {
- let router = UnfurlRouter::new(16);
+ let router = UnfurlRouter::new(16, test_lifetime());
let url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ";
router.l1_insert(&unfurl(url, None), &resolved_response());
assert!(
@@ -202,7 +235,7 @@ mod tests {
#[test]
fn invalidate_clears_every_provider_capability_variant() {
- let router = UnfurlRouter::new(16);
+ let router = UnfurlRouter::new(16, test_lifetime());
let url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ";
router.l1_insert(&unfurl(url, None), &resolved_response());
router.l1_insert(&unfurl(url, Some("configured")), &resolved_response());
@@ -223,4 +256,109 @@ mod tests {
UnfurlRouter::coalesce_key(&request(false))
);
}
+ #[test]
+ fn a_url_that_produced_no_embed_is_remembered_instead_of_being_resolved_again() {
+ let router = UnfurlRouter::new(16, test_lifetime());
+ let url = "https://example.com/no-metadata";
+ router.l1_insert(&unfurl(url, None), &empty_response());
+ assert!(
+ router.l1_lookup(&unfurl(url, None)).is_some(),
+ "an empty resolution must be served from the upper cache, not re-resolved"
+ );
+ }
+
+ #[test]
+ fn an_empty_cache_only_probe_is_never_written_to_the_upper_cache() {
+ let router = UnfurlRouter::new(16, test_lifetime());
+ let url = "https://example.com/never-probed-into-cache";
+ router.l1_insert(&probe(url), &empty_response());
+ assert!(
+ router.l1_lookup(&unfurl(url, None)).is_none(),
+ "a cache-only miss means unknown, not resolved-to-nothing"
+ );
+ }
+
+ #[test]
+ fn a_cache_only_probe_that_found_embeds_still_warms_the_upper_cache() {
+ let router = UnfurlRouter::new(16, test_lifetime());
+ let url = "https://example.com/warm";
+ router.l1_insert(&probe(url), &resolved_response());
+ assert!(router.l1_lookup(&unfurl(url, None)).is_some());
+ }
+
+ #[test]
+ fn configuring_a_klipy_key_does_not_reuse_the_keyless_cache_entry() {
+ let router = UnfurlRouter::new(16, test_lifetime());
+ let url = "https://klipy.com/gifs/example";
+ router.l1_insert(&unfurl_with_klipy(url, None), &resolved_response());
+ assert!(
+ router
+ .l1_lookup(&unfurl_with_klipy(url, Some("configured")))
+ .is_none(),
+ "a result resolved without a KLIPY key must not be served once a key is configured"
+ );
+ assert!(
+ router.l1_lookup(&unfurl_with_klipy(url, None)).is_some(),
+ "the keyless entry must still serve keyless requests"
+ );
+ }
+
+ #[test]
+ fn an_unscanned_entry_is_not_served_to_a_scanning_request() {
+ let router = UnfurlRouter::new(16, test_lifetime());
+ let url = "https://example.com/media";
+ router.l1_insert(&unfurl_allowing_nsfw(url), &resolved_response());
+ assert!(
+ router.l1_lookup(&unfurl(url, None)).is_none(),
+ "a result resolved with nsfw scanning off carries no nsfw flags and must not be reused"
+ );
+ assert!(router.l1_lookup(&unfurl_allowing_nsfw(url)).is_some());
+ }
+
+ #[test]
+ fn invalidate_clears_both_scanning_and_unscanned_entries() {
+ let router = UnfurlRouter::new(16, test_lifetime());
+ let url = "https://example.com/both";
+ router.l1_insert(&unfurl(url, None), &resolved_response());
+ router.l1_insert(&unfurl_allowing_nsfw(url), &resolved_response());
+ router.l1_insert(
+ &UnfurlRequest::Invalidate {
+ url: url.to_owned(),
+ },
+ &resolved_response(),
+ );
+ assert!(router.l1_lookup(&unfurl(url, None)).is_none());
+ assert!(router.l1_lookup(&unfurl_allowing_nsfw(url)).is_none());
+ }
+ #[tokio::test]
+ async fn an_empty_upper_cache_entry_lapses_long_before_a_populated_one() {
+ let router =
+ UnfurlRouter::with_empty_entry_lifetime(16, test_lifetime(), Duration::from_millis(50));
+ let empty_url = "https://example.com/transient";
+ let populated_url = "https://example.com/stable";
+ router.l1_insert(&unfurl(empty_url, None), &empty_response());
+ router.l1_insert(&unfurl(populated_url, None), &resolved_response());
+
+ tokio::time::sleep(Duration::from_millis(200)).await;
+
+ assert!(
+ router.l1_lookup(&unfurl(empty_url, None)).is_none(),
+ "a link that failed to unfurl must become retryable again quickly"
+ );
+ assert!(
+ router.l1_lookup(&unfurl(populated_url, None)).is_some(),
+ "a real embed must keep the full lifetime"
+ );
+ }
+ #[tokio::test]
+ async fn the_configured_lifetime_governs_the_upper_cache() {
+ let router = UnfurlRouter::new(16, Duration::from_millis(80));
+ let url = "https://example.com/configured";
+ router.l1_insert(&unfurl(url, None), &resolved_response());
+ tokio::time::sleep(Duration::from_millis(220)).await;
+ assert!(
+ router.l1_lookup(&unfurl(url, None)).is_none(),
+ "the configured lifetime must govern the upper cache"
+ );
+ }
}
diff --git a/fluxer_unfurl/src/shard_impl.rs b/fluxer_unfurl/src/shard_impl.rs
index 2ed8bf838..8333f3aa0 100644
--- a/fluxer_unfurl/src/shard_impl.rs
+++ b/fluxer_unfurl/src/shard_impl.rs
@@ -1,6 +1,9 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
-use crate::cache_policy::FIXED_UNFURL_CACHE_TTL_SECS;
+use crate::cache_policy::{
+ UnfurlEntryExpiry, default_empty_entry_lifetime, every_unfurl_cache_key, provider_capability,
+ unfurl_cache_key,
+};
use crate::embed_normalizer::normalize_embeds;
use crate::media_proxy::MediaProxyClient;
use crate::resolvers::{self, ResolveContext, ResolverResult};
@@ -12,10 +15,10 @@ use std::time::Duration;
use url::Url;
const L2_MAX_ENTRIES: u64 = 50_000;
-const L2_TTL: Duration = Duration::from_secs(FIXED_UNFURL_CACHE_TTL_SECS);
pub struct UnfurlShard {
cache: Cache<String, Arc<UnfurlResult>>,
+ lifetime: Duration,
http_client: reqwest::Client,
resolvers: Vec<Box<dyn crate::resolvers::Resolver>>,
media_proxy: MediaProxyClient,
@@ -23,7 +26,7 @@ pub struct UnfurlShard {
}
impl UnfurlShard {
- pub fn new() -> Self {
+ pub fn new(lifetime: Duration) -> Self {
let http_client = external_http_client();
let media_proxy_http_client = internal_http_client();
@@ -75,10 +78,8 @@ impl UnfurlShard {
);
Self {
- cache: Cache::builder()
- .max_capacity(L2_MAX_ENTRIES)
- .time_to_live(L2_TTL)
- .build(),
+ cache: build_l2_cache(lifetime, default_empty_entry_lifetime()),
+ lifetime,
http_client,
resolvers,
media_proxy,
@@ -86,6 +87,28 @@ impl UnfurlShard {
}
}
+ #[cfg(test)]
+ fn for_test() -> Self {
+ Self::for_test_with_lifetimes(long_lifetime(), default_empty_entry_lifetime())
+ }
+
+ #[cfg(test)]
+ fn for_test_with_lifetimes(lifetime: Duration, empty_entry_lifetime: Duration) -> Self {
+ Self {
+ cache: build_l2_cache(lifetime, empty_entry_lifetime),
+ lifetime,
+ http_client: external_http_client(),
+ resolvers: resolvers::build_resolver_chain(),
+ media_proxy: MediaProxyClient::new_with_public_endpoint(
+ "http://media-proxy.invalid",
+ "test-secret",
+ None,
+ internal_http_client(),
+ ),
+ static_cdn_endpoint: String::new(),
+ }
+ }
+
async fn resolve_url(
&self,
url_str: &str,
@@ -163,11 +186,27 @@ impl UnfurlShard {
fn finalize_result(&self, result: ResolverResult) -> UnfurlResult {
UnfurlResult {
embeds: normalize_embeds(result.embeds, &self.media_proxy),
- cache_ttl_seconds: Some(FIXED_UNFURL_CACHE_TTL_SECS),
+ cache_ttl_seconds: Some(self.lifetime.as_secs()),
}
}
}
+#[cfg(test)]
+fn long_lifetime() -> Duration {
+ Duration::from_secs(30 * 60)
+}
+
+fn build_l2_cache(
+ lifetime: Duration,
+ empty_entry_lifetime: Duration,
+) -> Cache<String, Arc<UnfurlResult>> {
+ Cache::builder()
+ .max_capacity(L2_MAX_ENTRIES)
+ .time_to_live(lifetime)
+ .expire_after(UnfurlEntryExpiry::new(empty_entry_lifetime))
+ .build()
+}
+
fn base_http_client_builder() -> reqwest::ClientBuilder {
reqwest::Client::builder()
.user_agent("Mozilla/5.0 (compatible; Fluxerbot/1.0; +https://fluxer.app)")
@@ -209,7 +248,11 @@ impl ShardService for UnfurlShard {
ref klipy_api_key,
} => {
let nsfw = nsfw_mode.unwrap_or_default();
- let cache_key = unfurl_cache_key(url, nsfw);
+ let cache_key = unfurl_cache_key(
+ url,
+ nsfw,
+ provider_capability(youtube_api_key.is_some(), klipy_api_key.is_some()),
+ );
if !bypass_cache && let Some(cached) = self.cache.get(&cache_key).await {
return Ok(UnfurlResponse::Resolved(cached));
@@ -239,21 +282,13 @@ impl ShardService for UnfurlShard {
}
};
- if !result.embeds.is_empty() {
- self.cache.insert(cache_key, result.clone()).await;
- }
+ self.cache.insert(cache_key, result.clone()).await;
Ok(UnfurlResponse::Resolved(result))
}
UnfurlRequest::Invalidate { ref url } => {
- self.cache
- .invalidate(&unfurl_cache_key(url, NsfwMode::Block))
- .await;
- self.cache
- .invalidate(&unfurl_cache_key(url, NsfwMode::Flag))
- .await;
- self.cache
- .invalidate(&unfurl_cache_key(url, NsfwMode::Allow))
- .await;
+ for key in every_unfurl_cache_key(url) {
+ self.cache.invalidate(&key).await;
+ }
Ok(UnfurlResponse::Invalidated(InvalidatedResponse {
invalidated: true,
}))
@@ -262,11 +297,230 @@ impl ShardService for UnfurlShard {
}
}
-fn unfurl_cache_key(url: &str, nsfw_mode: NsfwMode) -> String {
- let mode = match nsfw_mode {
- NsfwMode::Block => "block",
- NsfwMode::Flag => "flag",
- NsfwMode::Allow => "allow",
- };
- format!("{mode}:{url}")
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::types::MessageEmbed;
+
+ const BLOCKED_URL: &str = "http://127.0.0.1/unreachable";
+
+ fn unfurl(url: &str, youtube_api_key: Option<&str>) -> UnfurlRequest {
+ UnfurlRequest::Unfurl {
+ url: url.to_owned(),
+ nsfw_mode: Some(NsfwMode::Block),
+ bypass_cache: false,
+ cache_only: false,
+ youtube_api_key: youtube_api_key.map(str::to_owned),
+ klipy_api_key: None,
+ }
+ }
+
+ fn probe(url: &str) -> UnfurlRequest {
+ UnfurlRequest::Unfurl {
+ url: url.to_owned(),
+ nsfw_mode: Some(NsfwMode::Block),
+ bypass_cache: false,
+ cache_only: true,
+ youtube_api_key: None,
+ klipy_api_key: None,
+ }
+ }
+
+ fn keyless_block_key(url: &str) -> String {
+ unfurl_cache_key(url, NsfwMode::Block, provider_capability(false, false))
+ }
+
+ fn embed_count(response: &UnfurlResponse) -> usize {
+ match response {
+ UnfurlResponse::Resolved(result) => result.embeds.len(),
+ _ => panic!("expected a resolved response"),
+ }
+ }
+
+ async fn seed_keyless_entry(shard: &UnfurlShard) {
+ shard
+ .cache
+ .insert(
+ unfurl_cache_key(
+ BLOCKED_URL,
+ NsfwMode::Block,
+ provider_capability(false, false),
+ ),
+ Arc::new(UnfurlResult {
+ embeds: vec![MessageEmbed::new("link")],
+ cache_ttl_seconds: Some(long_lifetime().as_secs()),
+ }),
+ )
+ .await;
+ }
+
+ #[tokio::test]
+ async fn a_shard_entry_resolved_without_a_youtube_key_is_not_served_once_one_is_configured() {
+ let shard = UnfurlShard::for_test();
+ seed_keyless_entry(&shard).await;
+
+ assert_eq!(
+ 1,
+ embed_count(&shard.handle(unfurl(BLOCKED_URL, None)).await.unwrap()),
+ "the keyless entry must still serve keyless requests"
+ );
+ assert_eq!(
+ 0,
+ embed_count(
+ &shard
+ .handle(unfurl(BLOCKED_URL, Some("configured")))
+ .await
+ .unwrap()
+ ),
+ "a result resolved without a YouTube key must not be served once a key is configured"
+ );
+ }
+
+ #[tokio::test]
+ async fn invalidate_clears_the_shard_entry_for_every_provider_capability() {
+ let shard = UnfurlShard::for_test();
+ seed_keyless_entry(&shard).await;
+
+ shard
+ .handle(UnfurlRequest::Invalidate {
+ url: BLOCKED_URL.to_owned(),
+ })
+ .await
+ .unwrap();
+
+ assert_eq!(
+ 0,
+ embed_count(&shard.handle(unfurl(BLOCKED_URL, None)).await.unwrap())
+ );
+ }
+ #[tokio::test]
+ async fn a_url_that_produced_no_embed_is_written_to_the_shard_cache() {
+ let shard = UnfurlShard::for_test();
+
+ assert_eq!(
+ 0,
+ embed_count(&shard.handle(unfurl(BLOCKED_URL, None)).await.unwrap())
+ );
+ assert!(
+ shard
+ .cache
+ .get(&keyless_block_key(BLOCKED_URL))
+ .await
+ .is_some(),
+ "an empty resolution must be remembered so the next message does not refetch the page"
+ );
+ }
+
+ #[tokio::test]
+ async fn a_cache_only_probe_never_writes_an_entry() {
+ let shard = UnfurlShard::for_test();
+
+ assert_eq!(
+ 0,
+ embed_count(&shard.handle(probe(BLOCKED_URL)).await.unwrap())
+ );
+ assert!(
+ shard
+ .cache
+ .get(&keyless_block_key(BLOCKED_URL))
+ .await
+ .is_none(),
+ "a cache-only miss means unknown, not resolved-to-nothing"
+ );
+ }
+
+ #[tokio::test]
+ async fn an_unscanned_entry_is_not_served_to_a_scanning_request() {
+ let shard = UnfurlShard::for_test();
+ shard
+ .cache
+ .insert(
+ unfurl_cache_key(
+ BLOCKED_URL,
+ NsfwMode::Allow,
+ provider_capability(false, false),
+ ),
+ Arc::new(UnfurlResult {
+ embeds: vec![MessageEmbed::new("link")],
+ cache_ttl_seconds: Some(long_lifetime().as_secs()),
+ }),
+ )
+ .await;
+
+ assert_eq!(
+ 0,
+ embed_count(&shard.handle(unfurl(BLOCKED_URL, None)).await.unwrap()),
+ "a result resolved with nsfw scanning off must not be served to a scanning request"
+ );
+ }
+ #[tokio::test]
+ async fn an_empty_shard_entry_lapses_long_before_a_populated_one() {
+ let shard =
+ UnfurlShard::for_test_with_lifetimes(long_lifetime(), Duration::from_millis(50));
+ let empty_key = keyless_block_key("https://example.com/transient");
+ let populated_key = keyless_block_key("https://example.com/stable");
+ shard
+ .cache
+ .insert(
+ empty_key.clone(),
+ Arc::new(UnfurlResult {
+ embeds: Vec::new(),
+ cache_ttl_seconds: None,
+ }),
+ )
+ .await;
+ shard
+ .cache
+ .insert(
+ populated_key.clone(),
+ Arc::new(UnfurlResult {
+ embeds: vec![MessageEmbed::new("link")],
+ cache_ttl_seconds: Some(long_lifetime().as_secs()),
+ }),
+ )
+ .await;
+
+ tokio::time::sleep(Duration::from_millis(200)).await;
+
+ assert!(
+ shard.cache.get(&empty_key).await.is_none(),
+ "a link that failed to unfurl must become retryable again quickly"
+ );
+ assert!(
+ shard.cache.get(&populated_key).await.is_some(),
+ "a real embed must keep the full lifetime"
+ );
+ }
+ #[tokio::test]
+ async fn the_configured_lifetime_governs_the_shard_cache_and_the_reported_ttl() {
+ let shard = UnfurlShard::for_test_with_lifetimes(
+ Duration::from_millis(80),
+ default_empty_entry_lifetime(),
+ );
+ let finalized = shard.finalize_result(crate::resolvers::ResolverResult {
+ embeds: vec![MessageEmbed::new("link")],
+ });
+ assert_eq!(
+ Some(Duration::from_millis(80).as_secs()),
+ finalized.cache_ttl_seconds,
+ "the reported ttl must be the configured lifetime, not a compiled-in constant"
+ );
+
+ let key = keyless_block_key("https://example.com/configured");
+ shard
+ .cache
+ .insert(
+ key.clone(),
+ Arc::new(UnfurlResult {
+ embeds: vec![MessageEmbed::new("link")],
+ cache_ttl_seconds: None,
+ }),
+ )
+ .await;
+ tokio::time::sleep(Duration::from_millis(220)).await;
+ assert!(
+ shard.cache.get(&key).await.is_none(),
+ "the configured lifetime must govern the shard cache"
+ );
+ }
}