A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
refactor(api): resolve Bluesky identities in the unfurl service
Hampusbcb2ebeb6b8f
diff --git a/fluxer_api/src/api/bluesky/BlueskyOAuthService.ts b/fluxer_api/src/api/bluesky/BlueskyOAuthService.tsindex 018771242..1ac54a999 100644--- a/fluxer_api/src/api/bluesky/BlueskyOAuthService.ts+++ b/fluxer_api/src/api/bluesky/BlueskyOAuthService.ts@@ -19,10 +19,13 @@ import type {} from '@app/api/bluesky/IBlueskyOAuthService';import type {BlueskyOAuthConfig, BlueskyOAuthKeyConfig} from '@app/api/config/APIConfig';import {ConnectionCredentialRepository} from '@app/api/connection/ConnectionCredentialRepository';-import {Agent} from '@atproto/api';+import type {ExternalFetchResponse, IUnfurlerService} from '@app/api/infrastructure/IUnfurlerService';import {JoseKey} from '@atproto/jwk-jose';import {+ AtprotoHandleResolver,+ type Fetch,FetchError,+ type HandleResolver,NodeOAuthClient,type NodeSavedSessionStore,type OAuthClientMetadataInput,@@ -34,6 +37,43 @@ import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';interface BlueskyClientConfiguration {clientMetadata: OAuthClientMetadataInput;keyset: Array<JoseKey>;+ fetch: Fetch;+ handleResolver: HandleResolver;+}++const NULL_BODY_STATUSES = new Set([204, 205, 304]);++const HANDLE_LOOKUP_TIMEOUT_MS = 3000;+const HANDLE_INVALID = 'handle.invalid';++function createUnfurlerFetch(unfurler: IUnfurlerService): Fetch {+ return async (input, init) => {+ const request = new Request(input, init);+ request.signal.throwIfAborted();+ const body = request.method === 'GET' || request.method === 'HEAD' ? null : await request.text();+ const response = await new Promise<ExternalFetchResponse>((resolve, reject) => {+ const onAbort = () => reject(request.signal.reason);+ request.signal.addEventListener('abort', onAbort, {once: true});+ unfurler+ .fetchExternal({method: request.method, url: request.url, headers: [...request.headers], body})+ .then(resolve, (error: unknown) => reject(new TypeError('fetch failed', {cause: error})))+ .finally(() => request.signal.removeEventListener('abort', onAbort));+ });+ return new Response(NULL_BODY_STATUSES.has(response.status) ? null : response.body, {+ status: response.status,+ headers: response.headers,+ });+ };+}++function createHandleResolver(unfurler: IUnfurlerService, fetch: Fetch): HandleResolver {+ return new AtprotoHandleResolver({+ fetch: (input, init) => {+ const timeout = AbortSignal.timeout(HANDLE_LOOKUP_TIMEOUT_MS);+ return fetch(input, {...init, signal: init?.signal ? AbortSignal.any([init.signal, timeout]) : timeout});+ },+ resolveTxt: (name) => unfurler.resolveTxt(name),+ });}interface BlueskyClientSession {@@ -98,6 +138,7 @@ export class BlueskyOAuthService implements IBlueskyOAuthService {static async create(config: BlueskyOAuthConfig,kvClient: IKVProvider,+ unfurler: IUnfurlerService,apiPublicEndpoint: string,): Promise<BlueskyOAuthService> {const baseUrl = apiPublicEndpoint.replace(/\/$/, '');@@ -105,6 +146,8 @@ export class BlueskyOAuthService implements IBlueskyOAuthService {for (const [index, key] of config.keys.entries()) {keyset.push(await loadSigningKey(key, index));}+ const fetch = createUnfurlerFetch(unfurler);+ const handleResolver = createHandleResolver(unfurler, fetch);return new BlueskyOAuthService({clientMetadata: {@@ -125,6 +168,8 @@ export class BlueskyOAuthService implements IBlueskyOAuthService {jwks_uri: `${baseUrl}/connections/bluesky/jwks.json`,},keyset,+ fetch,+ handleResolver,},kvClient,);@@ -157,7 +202,7 @@ export class BlueskyOAuthService implements IBlueskyOAuthService {});const {session} = await client.callback(params);const grantOwner = requireOwner();- const handle = await this.resolveHandle(session.did);+ const handle = await this.resolveHandle(client, session.did);return {...grantOwner,did: session.did,@@ -182,21 +227,20 @@ export class BlueskyOAuthService implements IBlueskyOAuthService {return {client, sessionStore};}- private async resolveHandle(did: string): Promise<string> {- const agent = new Agent('https://public.api.bsky.app');+ private async resolveHandle(client: NodeOAuthClient, did: string): Promise<string> {try {- const profile = await agent.getProfile({actor: did});- return profile.data.handle;+ for (const noCache of [false, true]) {+ const {handle} = await client.identityResolver.resolve(did, {noCache});+ if (handle !== HANDLE_INVALID) return handle;+ }} catch (error) {- throw createOAuthOperationError('resolve profile', error);+ throw createOAuthOperationError('resolve handle', error);}+ throw createOAuthOperationError('resolve handle', new Error('The handle does not point back to the account'));}}-function createOAuthOperationError(- operation: 'authorize account' | 'restore session' | 'resolve profile',- error: unknown,-): Error {+function createOAuthOperationError(operation: 'authorize account' | 'resolve handle', error: unknown): Error {if (error instanceof BlueskyOAuthStoreError) return error;let status: number | undefined;if (error instanceof OAuthResponseError) status = error.status;diff --git a/fluxer_api/src/api/infrastructure/IUnfurlerService.ts b/fluxer_api/src/api/infrastructure/IUnfurlerService.tsindex a42dd6dbd..6a001da88 100644--- a/fluxer_api/src/api/infrastructure/IUnfurlerService.ts+++ b/fluxer_api/src/api/infrastructure/IUnfurlerService.ts@@ -14,6 +14,19 @@ export interface UnfurlOptions {cacheOnly?: boolean;}+export interface ExternalFetchRequest {+ method: string;+ url: string;+ headers: Array<[string, string]>;+ body: string | null;+}++export interface ExternalFetchResponse {+ status: number;+ headers: Array<[string, string]>;+ body: string;+}+export abstract class IUnfurlerService {async unfurl(url: string,@@ -30,4 +43,8 @@ export abstract class IUnfurlerService {): Promise<UnfurlResult>;abstract verifyDomain(domain: string, token: string): Promise<boolean>;++ abstract fetchExternal(request: ExternalFetchRequest): Promise<ExternalFetchResponse>;++ abstract resolveTxt(name: string): Promise<Array<string>>;}diff --git a/fluxer_api/src/api/infrastructure/NatsUnfurlerService.ts b/fluxer_api/src/api/infrastructure/NatsUnfurlerService.tsindex 1b8f2e33a..3fd7da392 100644--- a/fluxer_api/src/api/infrastructure/NatsUnfurlerService.ts+++ b/fluxer_api/src/api/infrastructure/NatsUnfurlerService.ts@@ -1,7 +1,13 @@// SPDX-License-Identifier: AGPL-3.0-or-laterimport type {MediaProxyNsfwMode} from '@app/api/infrastructure/IMediaService';-import {IUnfurlerService, type UnfurlOptions, type UnfurlResult} from '@app/api/infrastructure/IUnfurlerService';+import {+ type ExternalFetchRequest,+ type ExternalFetchResponse,+ IUnfurlerService,+ type UnfurlOptions,+ type UnfurlResult,+} from '@app/api/infrastructure/IUnfurlerService';import {requestSvcJson} from '@app/api/infrastructure/SvcErrorReply';import {Logger} from '@app/api/Logger';import {isJsonRecord} from '@app/api/utils/JsonBoundaryUtils';@@ -17,6 +23,8 @@ const NATS_UNFURL_SUBJECT = 'svc.unfurl';const NATS_UNFURL_TIMEOUT_MS = 12000;const NATS_UNFURL_CACHE_ONLY_TIMEOUT_MS = 1000;const NATS_VERIFY_DOMAIN_TIMEOUT_MS = 8000;+const NATS_FETCH_TIMEOUT_MS = 6000;+const NATS_RESOLVE_TXT_TIMEOUT_MS = 4000;interface NatsUnfurlRequest {op: 'Unfurl';@@ -72,6 +80,50 @@ function isNatsVerifyDomainResponse(value: unknown): value is NatsVerifyDomainRereturn false;}+type NatsFetchResponse = {Fetched: ExternalFetchResponse} | {Failed: {message: string}};++function isHeaderList(value: unknown): value is Array<[string, string]> {+ return (+ Array.isArray(value) &&+ value.every(+ (entry) =>+ Array.isArray(entry) && entry.length === 2 && typeof entry[0] === 'string' && typeof entry[1] === 'string',+ )+ );+}++function isNatsFetchResponse(value: unknown): value is NatsFetchResponse {+ if (!isJsonRecord(value)) return false;+ if ('Fetched' in value) {+ const fetched = value.Fetched;+ return (+ isJsonRecord(fetched) &&+ typeof fetched.status === 'number' &&+ isHeaderList(fetched.headers) &&+ typeof fetched.body === 'string'+ );+ }+ if ('Failed' in value) {+ const failed = value.Failed;+ return isJsonRecord(failed) && typeof failed.message === 'string';+ }+ return false;+}++type NatsTxtResponse = {Txt: Array<string>} | {Failed: {message: string}};++function isNatsTxtResponse(value: unknown): value is NatsTxtResponse {+ if (!isJsonRecord(value)) return false;+ if ('Txt' in value) {+ return Array.isArray(value.Txt) && value.Txt.every((record) => typeof record === 'string');+ }+ if ('Failed' in value) {+ const failed = value.Failed;+ return isJsonRecord(failed) && typeof failed.message === 'string';+ }+ return false;+}+function mapUnfurlTransportError(error: unknown): Error {if (error instanceof FluxerError) {return error;@@ -157,4 +209,44 @@ export class NatsUnfurlerService extends IUnfurlerService {throw mapUnfurlTransportError(error);}}++ async fetchExternal(request: ExternalFetchRequest): Promise<ExternalFetchResponse> {+ const response = await requestSvcJson(+ this.connectionManager,+ NATS_UNFURL_SUBJECT,+ {op: 'Fetch', ...request},+ {+ timeoutMs: NATS_FETCH_TIMEOUT_MS,+ guard: isNatsFetchResponse,+ service: 'nats-unfurl',+ invalidPayloadError: () => new BadGatewayError({message: '[nats-unfurl] invalid response payload'}),+ },+ ).catch((error: unknown) => {+ throw mapUnfurlTransportError(error);+ });+ if ('Fetched' in response) {+ return response.Fetched;+ }+ throw new BadGatewayError({message: `[nats-unfurl] fetch failed: ${response.Failed.message}`});+ }++ async resolveTxt(name: string): Promise<Array<string>> {+ const response = await requestSvcJson(+ this.connectionManager,+ NATS_UNFURL_SUBJECT,+ {op: 'ResolveTxt', name},+ {+ timeoutMs: NATS_RESOLVE_TXT_TIMEOUT_MS,+ guard: isNatsTxtResponse,+ service: 'nats-unfurl',+ invalidPayloadError: () => new BadGatewayError({message: '[nats-unfurl] invalid response payload'}),+ },+ ).catch((error: unknown) => {+ throw mapUnfurlTransportError(error);+ });+ if ('Txt' in response) {+ return response.Txt;+ }+ throw new BadGatewayError({message: `[nats-unfurl] TXT lookup failed: ${response.Failed.message}`});+ }}diff --git a/fluxer_api/src/api/middleware/ServiceMiddleware.ts b/fluxer_api/src/api/middleware/ServiceMiddleware.tsindex 6fab8166d..5248ba89b 100644--- a/fluxer_api/src/api/middleware/ServiceMiddleware.ts+++ b/fluxer_api/src/api/middleware/ServiceMiddleware.ts@@ -989,7 +989,7 @@ export const ServiceMiddleware = createMiddleware<HonoEnv>(async (ctx, next) =>ctx.set('sudoModeValid', false);getVirusScanServiceInstance();await ensureVoiceResourcesInitialized();- const blueskyOAuthService = await resolveBlueskyOAuthService(getInstanceConfigRepository());+ const blueskyOAuthService = await resolveBlueskyOAuthService(getUnfurlerService(), getInstanceConfigRepository());installLazyServices(ctx, new RequestServices(apiContext, blueskyOAuthService, ctx.get('requestCache')));await next();});diff --git a/fluxer_api/src/api/middleware/ServiceRegistry.ts b/fluxer_api/src/api/middleware/ServiceRegistry.tsindex 29e3f7c82..2e552685e 100644--- a/fluxer_api/src/api/middleware/ServiceRegistry.ts+++ b/fluxer_api/src/api/middleware/ServiceRegistry.ts@@ -1,7 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-laterimport {BillingRepository} from '@app/api/billing/repositories/BillingRepository';-import {BlueskyOAuthService} from '@app/api/bluesky/BlueskyOAuthService';import type {IBlueskyOAuthService} from '@app/api/bluesky/IBlueskyOAuthService';import {Config} from '@app/api/Config';import type {BlueskyOAuthConfig} from '@app/api/config/APIConfig';@@ -12,6 +11,7 @@ import type {ILiveKitService} from '@app/api/infrastructure/ILiveKitService';import type {IMediaService} from '@app/api/infrastructure/IMediaService';import {InMemoryVoiceRoomStore} from '@app/api/infrastructure/InMemoryVoiceRoomStore';import type {ISnowflakeService} from '@app/api/infrastructure/ISnowflakeService';+import type {IUnfurlerService} from '@app/api/infrastructure/IUnfurlerService';import type {IVoiceRoomStore} from '@app/api/infrastructure/IVoiceRoomStore';import {LiveKitService} from '@app/api/infrastructure/LiveKitService';import {MediaService as ProdMediaService} from '@app/api/infrastructure/MediaService';@@ -171,14 +171,17 @@ function resetBlueskyOAuthServiceCache(): void {_blueskyOAuthGeneration = Symbol();}-async function createBlueskyOAuthService(config: BlueskyOAuthConfig): Promise<IBlueskyOAuthService | null> {+async function createBlueskyOAuthService(+ config: BlueskyOAuthConfig,+ unfurler: IUnfurlerService,+): Promise<IBlueskyOAuthService | null> {if (!config.enabled) return null;if (config.keys.length === 0) {Logger.warn('Bluesky OAuth is enabled but no signing keys are configured; disabling Bluesky OAuth.');return null;}try {- return await BlueskyOAuthService.create(config, getKVClient(), Config.endpoints.apiPublic);+ return await BlueskyOAuthService.create(config, getKVClient(), unfurler, Config.endpoints.apiPublic);} catch (error) {Logger.error({error}, 'Bluesky OAuth initialization failed; disabling Bluesky OAuth for this configuration.');return null;@@ -186,6 +189,7 @@ async function createBlueskyOAuthService(config: BlueskyOAuthConfig): Promise<IB}export async function resolveBlueskyOAuthService(+ unfurler: IUnfurlerService,instanceConfigRepository?: Pick<InstanceConfigRepository, 'getEffectiveBlueskyConfig'>,): Promise<IBlueskyOAuthService | null> {if (_injectedBlueskyOAuthService) {@@ -208,7 +212,7 @@ export async function resolveBlueskyOAuthService(const next: BlueskyOAuthServiceSelection = {config: blueskyConfig,signature,- service: createBlueskyOAuthService(blueskyConfig),+ service: createBlueskyOAuthService(blueskyConfig, unfurler),};_blueskyOAuthSelection = next;return next.service;diff --git a/fluxer_api/src/api/test/mocks/NoopUnfurlerService.ts b/fluxer_api/src/api/test/mocks/NoopUnfurlerService.tsindex c55df3504..f2e7722ee 100644--- a/fluxer_api/src/api/test/mocks/NoopUnfurlerService.ts+++ b/fluxer_api/src/api/test/mocks/NoopUnfurlerService.ts@@ -1,7 +1,13 @@// SPDX-License-Identifier: AGPL-3.0-or-laterimport type {MediaProxyNsfwMode} from '@app/api/infrastructure/IMediaService';-import {IUnfurlerService, type UnfurlOptions, type UnfurlResult} from '@app/api/infrastructure/IUnfurlerService';+import {+ type ExternalFetchRequest,+ type ExternalFetchResponse,+ IUnfurlerService,+ type UnfurlOptions,+ type UnfurlResult,+} from '@app/api/infrastructure/IUnfurlerService';export class NoopUnfurlerService extends IUnfurlerService {override async unfurlWithCachePolicy(@@ -15,4 +21,12 @@ export class NoopUnfurlerService extends IUnfurlerService {override async verifyDomain(_domain: string, _token: string): Promise<boolean> {return false;}++ override async fetchExternal(_request: ExternalFetchRequest): Promise<ExternalFetchResponse> {+ throw new Error('External fetches are unavailable');+ }++ override async resolveTxt(_name: string): Promise<Array<string>> {+ return [];+ }}diff --git a/fluxer_unfurl/src/external_fetch.rs b/fluxer_unfurl/src/external_fetch.rsnew file mode 100644index 000000000..46cac78a6--- /dev/null+++ b/fluxer_unfurl/src/external_fetch.rs@@ -0,0 +1,156 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++use crate::{http_fetch, network_policy};+use reqwest::Method;+use reqwest::header::{+ ACCEPT, ACCEPT_ENCODING, AUTHORIZATION, CONTENT_ENCODING, CONTENT_TYPE, HeaderMap, HeaderName,+ HeaderValue,+};+use serde::{Deserialize, Serialize};+use std::time::Duration;+use tokio::time::timeout;+use url::Url;++const FETCH_TIMEOUT: Duration = Duration::from_secs(4);+const MAX_RESPONSE_BYTES: usize = 256 * 1024;+const MAX_REQUEST_BODY_BYTES: usize = 64 * 1024;+const DPOP: HeaderName = HeaderName::from_static("dpop");++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct FetchedResponse {+ pub status: u16,+ pub headers: Vec<(String, String)>,+ pub body: String,+}++pub async fn fetch(+ client: &reqwest::Client,+ method: &str,+ url: &str,+ headers: &[(String, String)],+ body: Option<String>,+) -> anyhow::Result<FetchedResponse> {+ let method = match method {+ "GET" => Method::GET,+ "POST" => Method::POST,+ _ => anyhow::bail!("unsupported method"),+ };+ let url = network_policy::parse_url(url)?;+ validate_target(&url)?;+ if body+ .as_ref()+ .is_some_and(|b| b.len() > MAX_REQUEST_BODY_BYTES)+ {+ anyhow::bail!("request body too large");+ }+ let headers = forwarded_headers(headers)?;+ timeout(FETCH_TIMEOUT, async move {+ network_policy::validate_url(&url).await?;+ let mut request = client.request(method, url).headers(headers);+ if let Some(body) = body {+ request = request.body(body);+ }+ let response = request.send().await?;+ let status = response.status().as_u16();+ if let Some(encoding) = response.headers().get(CONTENT_ENCODING)+ && !encoding+ .to_str()+ .is_ok_and(|value| value.trim().eq_ignore_ascii_case("identity"))+ {+ anyhow::bail!("encoded response bodies are not accepted");+ }+ if response+ .content_length()+ .is_some_and(|len| len > MAX_RESPONSE_BYTES as u64)+ {+ anyhow::bail!("response body too large");+ }+ let headers = response+ .headers()+ .iter()+ .filter_map(|(name, value)| {+ value+ .to_str()+ .ok()+ .map(|value| (name.as_str().to_owned(), value.to_owned()))+ })+ .collect();+ let bytes = http_fetch::read_body(response, MAX_RESPONSE_BYTES).await?;+ Ok(FetchedResponse {+ status,+ headers,+ body: String::from_utf8(bytes)+ .map_err(|_| anyhow::anyhow!("response body is not UTF-8"))?,+ })+ })+ .await+ .map_err(|_| anyhow::anyhow!("request timed out"))?+}++fn validate_target(url: &Url) -> anyhow::Result<()> {+ if url.scheme() != "https" || url.port().is_some() || url.fragment().is_some() {+ anyhow::bail!("blocked URL");+ }+ if !matches!(url.host(), Some(url::Host::Domain(_))) {+ anyhow::bail!("blocked URL");+ }+ Ok(())+}++fn forwarded_headers(headers: &[(String, String)]) -> anyhow::Result<HeaderMap> {+ let mut forwarded = HeaderMap::new();+ for (name, value) in headers {+ let name = HeaderName::from_bytes(name.as_bytes())?;+ if [ACCEPT, AUTHORIZATION, CONTENT_TYPE, DPOP].contains(&name) {+ forwarded.append(name, HeaderValue::from_str(value)?);+ }+ }+ forwarded.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity"));+ Ok(forwarded)+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn only_https_hostnames_on_the_default_port() {+ for raw in [+ "https://bsky.social/xrpc/x",+ "https://plc.directory/did:plc:abc",+ ] {+ assert!(validate_target(&Url::parse(raw).unwrap()).is_ok(), "{raw}");+ }+ for raw in [+ "http://bsky.social/",+ "https://bsky.social:8443/",+ "https://1.1.1.1/",+ "https://[2606:4700::1111]/",+ "https://bsky.social/#x",+ ] {+ assert!(validate_target(&Url::parse(raw).unwrap()).is_err(), "{raw}");+ }+ }++ #[test]+ fn forwards_only_known_headers_and_asks_for_identity() {+ let forwarded = forwarded_headers(&[+ ("Accept".into(), "application/json".into()),+ ("DPoP".into(), "proof".into()),+ ("Authorization".into(), "DPoP token".into()),+ (+ "Content-Type".into(),+ "application/x-www-form-urlencoded".into(),+ ),+ ("Host".into(), "internal".into()),+ ("Accept-Encoding".into(), "br".into()),+ ("Cookie".into(), "a=b".into()),+ ])+ .unwrap();+ assert_eq!(forwarded.len(), 5);+ assert_eq!(forwarded[ACCEPT_ENCODING], "identity");+ assert_eq!(forwarded[DPOP], "proof");+ assert!(forwarded.get("host").is_none());+ assert!(forwarded.get("cookie").is_none());+ }+}diff --git a/fluxer_unfurl/src/main.rs b/fluxer_unfurl/src/main.rsindex 5804c28ff..7e1e81248 100644--- a/fluxer_unfurl/src/main.rs+++ b/fluxer_unfurl/src/main.rs@@ -7,6 +7,7 @@ mod direct_media;mod dns_txt;mod domain_verification;mod embed_normalizer;+mod external_fetch;mod html_markdown;mod html_parser;mod http_fetch;diff --git a/fluxer_unfurl/src/router_impl.rs b/fluxer_unfurl/src/router_impl.rsindex e2080e9f2..54d304f46 100644--- a/fluxer_unfurl/src/router_impl.rs+++ b/fluxer_unfurl/src/router_impl.rs@@ -44,6 +44,8 @@ impl RouterService for UnfurlRouter {match req {UnfurlRequest::Unfurl { url, .. } => url.clone(),UnfurlRequest::VerifyDomain { domain, .. } => domain.clone(),+ UnfurlRequest::Fetch { url, .. } => url.clone(),+ UnfurlRequest::ResolveTxt { name } => name.clone(),}}@@ -70,7 +72,9 @@ impl RouterService for UnfurlRouter {)))}- UnfurlRequest::VerifyDomain { .. } => None,+ UnfurlRequest::VerifyDomain { .. }+ | UnfurlRequest::Fetch { .. }+ | UnfurlRequest::ResolveTxt { .. } => None,}}@@ -95,7 +99,9 @@ impl RouterService for UnfurlRouter {self.l1.get(&key)}}- UnfurlRequest::VerifyDomain { .. } => None,+ UnfurlRequest::VerifyDomain { .. }+ | UnfurlRequest::Fetch { .. }+ | UnfurlRequest::ResolveTxt { .. } => None,}}@@ -122,7 +128,9 @@ impl RouterService for UnfurlRouter {);}}- UnfurlRequest::VerifyDomain { .. } => {}+ UnfurlRequest::VerifyDomain { .. }+ | UnfurlRequest::Fetch { .. }+ | UnfurlRequest::ResolveTxt { .. } => {}}}}diff --git a/fluxer_unfurl/src/shard_impl.rs b/fluxer_unfurl/src/shard_impl.rsindex 54d753c13..86a6e0b15 100644--- a/fluxer_unfurl/src/shard_impl.rs+++ b/fluxer_unfurl/src/shard_impl.rs@@ -211,6 +211,10 @@ fn base_http_client_builder() -> reqwest::ClientBuilder {.user_agent("Mozilla/5.0 (compatible; Fluxerbot/1.0; +https://fluxer.app)").timeout(Duration::from_secs(10)).redirect(reqwest::redirect::Policy::none())+ .no_gzip()+ .no_brotli()+ .no_deflate()+ .no_zstd()}fn external_http_client() -> reqwest::Client {@@ -294,6 +298,32 @@ impl ShardService for UnfurlShard {}),}}+ UnfurlRequest::Fetch {+ method,+ url,+ headers,+ body,+ } => {+ match crate::external_fetch::fetch(&self.http_client, &method, &url, &headers, body)+ .await+ {+ Ok(response) => Ok(UnfurlResponse::Fetched(response)),+ Err(err) => Ok(UnfurlResponse::Failed {+ message: err.to_string(),+ }),+ }+ }+ UnfurlRequest::ResolveTxt { name } => {+ if let Err(err) = crate::dns_txt::validate_record_name(&name) {+ return Ok(UnfurlResponse::Failed {+ message: err.to_string(),+ });+ }+ let records = crate::dns_txt::first_answer(&name, |records| !records.is_empty())+ .await+ .unwrap_or_default();+ Ok(UnfurlResponse::Txt(records))+ }}}}diff --git a/fluxer_unfurl/src/types.rs b/fluxer_unfurl/src/types.rsindex cd388eb65..70c5ed823 100644--- a/fluxer_unfurl/src/types.rs+++ b/fluxer_unfurl/src/types.rs@@ -1,5 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later+use crate::external_fetch::FetchedResponse;use serde::{Deserialize, Serialize};use std::sync::Arc;@@ -22,6 +23,17 @@ pub enum UnfurlRequest {domain: String,token: String,},+ Fetch {+ method: String,+ url: String,+ #[serde(default)]+ headers: Vec<(String, String)>,+ #[serde(default)]+ body: Option<String>,+ },+ ResolveTxt {+ name: String,+ },}#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]@@ -37,6 +49,8 @@ pub enum NsfwMode {pub enum UnfurlResponse {Resolved(Arc<UnfurlResult>),Verified(bool),+ Fetched(FetchedResponse),+ Txt(Vec<String>),Failed { message: String },}@@ -209,7 +223,7 @@ mod tests {assert!(youtube_api_key.is_none());assert!(klipy_api_key.is_none());}- UnfurlRequest::VerifyDomain { .. } => panic!("expected an unfurl request"),+ _ => panic!("expected an unfurl request"),}}}