Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

refactor(api): resolve Bluesky identities in the unfurl service

Hampusbcb2ebeb6b8f
diff --git a/fluxer_api/src/api/bluesky/BlueskyOAuthService.ts b/fluxer_api/src/api/bluesky/BlueskyOAuthService.ts
index 018771242..1ac54a999 100644
--- a/fluxer_api/src/api/bluesky/BlueskyOAuthService.ts
+++ b/fluxer_api/src/api/bluesky/BlueskyOAuthService.ts
@@ -19,10 +19,13 @@ import type {
} from '@app/api/bluesky/IBlueskyOAuthService';
import type {BlueskyOAuthConfig, BlueskyOAuthKeyConfig} from '@app/api/config/APIConfig';
import {ConnectionCredentialRepository} from '@app/api/connection/ConnectionCredentialRepository';
-import {Agent} from '@atproto/api';
+import type {ExternalFetchResponse, IUnfurlerService} from '@app/api/infrastructure/IUnfurlerService';
import {JoseKey} from '@atproto/jwk-jose';
import {
+ AtprotoHandleResolver,
+ type Fetch,
FetchError,
+ type HandleResolver,
NodeOAuthClient,
type NodeSavedSessionStore,
type OAuthClientMetadataInput,
@@ -34,6 +37,43 @@ import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
interface BlueskyClientConfiguration {
clientMetadata: OAuthClientMetadataInput;
keyset: Array<JoseKey>;
+ fetch: Fetch;
+ handleResolver: HandleResolver;
+}
+
+const NULL_BODY_STATUSES = new Set([204, 205, 304]);
+
+const HANDLE_LOOKUP_TIMEOUT_MS = 3000;
+const HANDLE_INVALID = 'handle.invalid';
+
+function createUnfurlerFetch(unfurler: IUnfurlerService): Fetch {
+ return async (input, init) => {
+ const request = new Request(input, init);
+ request.signal.throwIfAborted();
+ const body = request.method === 'GET' || request.method === 'HEAD' ? null : await request.text();
+ const response = await new Promise<ExternalFetchResponse>((resolve, reject) => {
+ const onAbort = () => reject(request.signal.reason);
+ request.signal.addEventListener('abort', onAbort, {once: true});
+ unfurler
+ .fetchExternal({method: request.method, url: request.url, headers: [...request.headers], body})
+ .then(resolve, (error: unknown) => reject(new TypeError('fetch failed', {cause: error})))
+ .finally(() => request.signal.removeEventListener('abort', onAbort));
+ });
+ return new Response(NULL_BODY_STATUSES.has(response.status) ? null : response.body, {
+ status: response.status,
+ headers: response.headers,
+ });
+ };
+}
+
+function createHandleResolver(unfurler: IUnfurlerService, fetch: Fetch): HandleResolver {
+ return new AtprotoHandleResolver({
+ fetch: (input, init) => {
+ const timeout = AbortSignal.timeout(HANDLE_LOOKUP_TIMEOUT_MS);
+ return fetch(input, {...init, signal: init?.signal ? AbortSignal.any([init.signal, timeout]) : timeout});
+ },
+ resolveTxt: (name) => unfurler.resolveTxt(name),
+ });
}
interface BlueskyClientSession {
@@ -98,6 +138,7 @@ export class BlueskyOAuthService implements IBlueskyOAuthService {
static async create(
config: BlueskyOAuthConfig,
kvClient: IKVProvider,
+ unfurler: IUnfurlerService,
apiPublicEndpoint: string,
): Promise<BlueskyOAuthService> {
const baseUrl = apiPublicEndpoint.replace(/\/$/, '');
@@ -105,6 +146,8 @@ export class BlueskyOAuthService implements IBlueskyOAuthService {
for (const [index, key] of config.keys.entries()) {
keyset.push(await loadSigningKey(key, index));
}
+ const fetch = createUnfurlerFetch(unfurler);
+ const handleResolver = createHandleResolver(unfurler, fetch);
return new BlueskyOAuthService(
{
clientMetadata: {
@@ -125,6 +168,8 @@ export class BlueskyOAuthService implements IBlueskyOAuthService {
jwks_uri: `${baseUrl}/connections/bluesky/jwks.json`,
},
keyset,
+ fetch,
+ handleResolver,
},
kvClient,
);
@@ -157,7 +202,7 @@ export class BlueskyOAuthService implements IBlueskyOAuthService {
});
const {session} = await client.callback(params);
const grantOwner = requireOwner();
- const handle = await this.resolveHandle(session.did);
+ const handle = await this.resolveHandle(client, session.did);
return {
...grantOwner,
did: session.did,
@@ -182,21 +227,20 @@ export class BlueskyOAuthService implements IBlueskyOAuthService {
return {client, sessionStore};
}
- private async resolveHandle(did: string): Promise<string> {
- const agent = new Agent('https://public.api.bsky.app');
+ private async resolveHandle(client: NodeOAuthClient, did: string): Promise<string> {
try {
- const profile = await agent.getProfile({actor: did});
- return profile.data.handle;
+ for (const noCache of [false, true]) {
+ const {handle} = await client.identityResolver.resolve(did, {noCache});
+ if (handle !== HANDLE_INVALID) return handle;
+ }
} catch (error) {
- throw createOAuthOperationError('resolve profile', error);
+ throw createOAuthOperationError('resolve handle', error);
}
+ throw createOAuthOperationError('resolve handle', new Error('The handle does not point back to the account'));
}
}
-function createOAuthOperationError(
- operation: 'authorize account' | 'restore session' | 'resolve profile',
- error: unknown,
-): Error {
+function createOAuthOperationError(operation: 'authorize account' | 'resolve handle', error: unknown): Error {
if (error instanceof BlueskyOAuthStoreError) return error;
let status: number | undefined;
if (error instanceof OAuthResponseError) status = error.status;
diff --git a/fluxer_api/src/api/infrastructure/IUnfurlerService.ts b/fluxer_api/src/api/infrastructure/IUnfurlerService.ts
index a42dd6dbd..6a001da88 100644
--- a/fluxer_api/src/api/infrastructure/IUnfurlerService.ts
+++ b/fluxer_api/src/api/infrastructure/IUnfurlerService.ts
@@ -14,6 +14,19 @@ export interface UnfurlOptions {
cacheOnly?: boolean;
}
+export interface ExternalFetchRequest {
+ method: string;
+ url: string;
+ headers: Array<[string, string]>;
+ body: string | null;
+}
+
+export interface ExternalFetchResponse {
+ status: number;
+ headers: Array<[string, string]>;
+ body: string;
+}
+
export abstract class IUnfurlerService {
async unfurl(
url: string,
@@ -30,4 +43,8 @@ export abstract class IUnfurlerService {
): Promise<UnfurlResult>;
abstract verifyDomain(domain: string, token: string): Promise<boolean>;
+
+ abstract fetchExternal(request: ExternalFetchRequest): Promise<ExternalFetchResponse>;
+
+ abstract resolveTxt(name: string): Promise<Array<string>>;
}
diff --git a/fluxer_api/src/api/infrastructure/NatsUnfurlerService.ts b/fluxer_api/src/api/infrastructure/NatsUnfurlerService.ts
index 1b8f2e33a..3fd7da392 100644
--- a/fluxer_api/src/api/infrastructure/NatsUnfurlerService.ts
+++ b/fluxer_api/src/api/infrastructure/NatsUnfurlerService.ts
@@ -1,7 +1,13 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {MediaProxyNsfwMode} from '@app/api/infrastructure/IMediaService';
-import {IUnfurlerService, type UnfurlOptions, type UnfurlResult} from '@app/api/infrastructure/IUnfurlerService';
+import {
+ type ExternalFetchRequest,
+ type ExternalFetchResponse,
+ IUnfurlerService,
+ type UnfurlOptions,
+ type UnfurlResult,
+} from '@app/api/infrastructure/IUnfurlerService';
import {requestSvcJson} from '@app/api/infrastructure/SvcErrorReply';
import {Logger} from '@app/api/Logger';
import {isJsonRecord} from '@app/api/utils/JsonBoundaryUtils';
@@ -17,6 +23,8 @@ const NATS_UNFURL_SUBJECT = 'svc.unfurl';
const NATS_UNFURL_TIMEOUT_MS = 12000;
const NATS_UNFURL_CACHE_ONLY_TIMEOUT_MS = 1000;
const NATS_VERIFY_DOMAIN_TIMEOUT_MS = 8000;
+const NATS_FETCH_TIMEOUT_MS = 6000;
+const NATS_RESOLVE_TXT_TIMEOUT_MS = 4000;
interface NatsUnfurlRequest {
op: 'Unfurl';
@@ -72,6 +80,50 @@ function isNatsVerifyDomainResponse(value: unknown): value is NatsVerifyDomainRe
return false;
}
+type NatsFetchResponse = {Fetched: ExternalFetchResponse} | {Failed: {message: string}};
+
+function isHeaderList(value: unknown): value is Array<[string, string]> {
+ return (
+ Array.isArray(value) &&
+ value.every(
+ (entry) =>
+ Array.isArray(entry) && entry.length === 2 && typeof entry[0] === 'string' && typeof entry[1] === 'string',
+ )
+ );
+}
+
+function isNatsFetchResponse(value: unknown): value is NatsFetchResponse {
+ if (!isJsonRecord(value)) return false;
+ if ('Fetched' in value) {
+ const fetched = value.Fetched;
+ return (
+ isJsonRecord(fetched) &&
+ typeof fetched.status === 'number' &&
+ isHeaderList(fetched.headers) &&
+ typeof fetched.body === 'string'
+ );
+ }
+ if ('Failed' in value) {
+ const failed = value.Failed;
+ return isJsonRecord(failed) && typeof failed.message === 'string';
+ }
+ return false;
+}
+
+type NatsTxtResponse = {Txt: Array<string>} | {Failed: {message: string}};
+
+function isNatsTxtResponse(value: unknown): value is NatsTxtResponse {
+ if (!isJsonRecord(value)) return false;
+ if ('Txt' in value) {
+ return Array.isArray(value.Txt) && value.Txt.every((record) => typeof record === 'string');
+ }
+ if ('Failed' in value) {
+ const failed = value.Failed;
+ return isJsonRecord(failed) && typeof failed.message === 'string';
+ }
+ return false;
+}
+
function mapUnfurlTransportError(error: unknown): Error {
if (error instanceof FluxerError) {
return error;
@@ -157,4 +209,44 @@ export class NatsUnfurlerService extends IUnfurlerService {
throw mapUnfurlTransportError(error);
}
}
+
+ async fetchExternal(request: ExternalFetchRequest): Promise<ExternalFetchResponse> {
+ const response = await requestSvcJson(
+ this.connectionManager,
+ NATS_UNFURL_SUBJECT,
+ {op: 'Fetch', ...request},
+ {
+ timeoutMs: NATS_FETCH_TIMEOUT_MS,
+ guard: isNatsFetchResponse,
+ service: 'nats-unfurl',
+ invalidPayloadError: () => new BadGatewayError({message: '[nats-unfurl] invalid response payload'}),
+ },
+ ).catch((error: unknown) => {
+ throw mapUnfurlTransportError(error);
+ });
+ if ('Fetched' in response) {
+ return response.Fetched;
+ }
+ throw new BadGatewayError({message: `[nats-unfurl] fetch failed: ${response.Failed.message}`});
+ }
+
+ async resolveTxt(name: string): Promise<Array<string>> {
+ const response = await requestSvcJson(
+ this.connectionManager,
+ NATS_UNFURL_SUBJECT,
+ {op: 'ResolveTxt', name},
+ {
+ timeoutMs: NATS_RESOLVE_TXT_TIMEOUT_MS,
+ guard: isNatsTxtResponse,
+ service: 'nats-unfurl',
+ invalidPayloadError: () => new BadGatewayError({message: '[nats-unfurl] invalid response payload'}),
+ },
+ ).catch((error: unknown) => {
+ throw mapUnfurlTransportError(error);
+ });
+ if ('Txt' in response) {
+ return response.Txt;
+ }
+ throw new BadGatewayError({message: `[nats-unfurl] TXT lookup failed: ${response.Failed.message}`});
+ }
}
diff --git a/fluxer_api/src/api/middleware/ServiceMiddleware.ts b/fluxer_api/src/api/middleware/ServiceMiddleware.ts
index 6fab8166d..5248ba89b 100644
--- a/fluxer_api/src/api/middleware/ServiceMiddleware.ts
+++ b/fluxer_api/src/api/middleware/ServiceMiddleware.ts
@@ -989,7 +989,7 @@ export const ServiceMiddleware = createMiddleware<HonoEnv>(async (ctx, next) =>
ctx.set('sudoModeValid', false);
getVirusScanServiceInstance();
await ensureVoiceResourcesInitialized();
- const blueskyOAuthService = await resolveBlueskyOAuthService(getInstanceConfigRepository());
+ const blueskyOAuthService = await resolveBlueskyOAuthService(getUnfurlerService(), getInstanceConfigRepository());
installLazyServices(ctx, new RequestServices(apiContext, blueskyOAuthService, ctx.get('requestCache')));
await next();
});
diff --git a/fluxer_api/src/api/middleware/ServiceRegistry.ts b/fluxer_api/src/api/middleware/ServiceRegistry.ts
index 29e3f7c82..2e552685e 100644
--- a/fluxer_api/src/api/middleware/ServiceRegistry.ts
+++ b/fluxer_api/src/api/middleware/ServiceRegistry.ts
@@ -1,7 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {BillingRepository} from '@app/api/billing/repositories/BillingRepository';
-import {BlueskyOAuthService} from '@app/api/bluesky/BlueskyOAuthService';
import type {IBlueskyOAuthService} from '@app/api/bluesky/IBlueskyOAuthService';
import {Config} from '@app/api/Config';
import type {BlueskyOAuthConfig} from '@app/api/config/APIConfig';
@@ -12,6 +11,7 @@ import type {ILiveKitService} from '@app/api/infrastructure/ILiveKitService';
import type {IMediaService} from '@app/api/infrastructure/IMediaService';
import {InMemoryVoiceRoomStore} from '@app/api/infrastructure/InMemoryVoiceRoomStore';
import type {ISnowflakeService} from '@app/api/infrastructure/ISnowflakeService';
+import type {IUnfurlerService} from '@app/api/infrastructure/IUnfurlerService';
import type {IVoiceRoomStore} from '@app/api/infrastructure/IVoiceRoomStore';
import {LiveKitService} from '@app/api/infrastructure/LiveKitService';
import {MediaService as ProdMediaService} from '@app/api/infrastructure/MediaService';
@@ -171,14 +171,17 @@ function resetBlueskyOAuthServiceCache(): void {
_blueskyOAuthGeneration = Symbol();
}
-async function createBlueskyOAuthService(config: BlueskyOAuthConfig): Promise<IBlueskyOAuthService | null> {
+async function createBlueskyOAuthService(
+ config: BlueskyOAuthConfig,
+ unfurler: IUnfurlerService,
+): Promise<IBlueskyOAuthService | null> {
if (!config.enabled) return null;
if (config.keys.length === 0) {
Logger.warn('Bluesky OAuth is enabled but no signing keys are configured; disabling Bluesky OAuth.');
return null;
}
try {
- return await BlueskyOAuthService.create(config, getKVClient(), Config.endpoints.apiPublic);
+ return await BlueskyOAuthService.create(config, getKVClient(), unfurler, Config.endpoints.apiPublic);
} catch (error) {
Logger.error({error}, 'Bluesky OAuth initialization failed; disabling Bluesky OAuth for this configuration.');
return null;
@@ -186,6 +189,7 @@ async function createBlueskyOAuthService(config: BlueskyOAuthConfig): Promise<IB
}
export async function resolveBlueskyOAuthService(
+ unfurler: IUnfurlerService,
instanceConfigRepository?: Pick<InstanceConfigRepository, 'getEffectiveBlueskyConfig'>,
): Promise<IBlueskyOAuthService | null> {
if (_injectedBlueskyOAuthService) {
@@ -208,7 +212,7 @@ export async function resolveBlueskyOAuthService(
const next: BlueskyOAuthServiceSelection = {
config: blueskyConfig,
signature,
- service: createBlueskyOAuthService(blueskyConfig),
+ service: createBlueskyOAuthService(blueskyConfig, unfurler),
};
_blueskyOAuthSelection = next;
return next.service;
diff --git a/fluxer_api/src/api/test/mocks/NoopUnfurlerService.ts b/fluxer_api/src/api/test/mocks/NoopUnfurlerService.ts
index c55df3504..f2e7722ee 100644
--- a/fluxer_api/src/api/test/mocks/NoopUnfurlerService.ts
+++ b/fluxer_api/src/api/test/mocks/NoopUnfurlerService.ts
@@ -1,7 +1,13 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {MediaProxyNsfwMode} from '@app/api/infrastructure/IMediaService';
-import {IUnfurlerService, type UnfurlOptions, type UnfurlResult} from '@app/api/infrastructure/IUnfurlerService';
+import {
+ type ExternalFetchRequest,
+ type ExternalFetchResponse,
+ IUnfurlerService,
+ type UnfurlOptions,
+ type UnfurlResult,
+} from '@app/api/infrastructure/IUnfurlerService';
export class NoopUnfurlerService extends IUnfurlerService {
override async unfurlWithCachePolicy(
@@ -15,4 +21,12 @@ export class NoopUnfurlerService extends IUnfurlerService {
override async verifyDomain(_domain: string, _token: string): Promise<boolean> {
return false;
}
+
+ override async fetchExternal(_request: ExternalFetchRequest): Promise<ExternalFetchResponse> {
+ throw new Error('External fetches are unavailable');
+ }
+
+ override async resolveTxt(_name: string): Promise<Array<string>> {
+ return [];
+ }
}
diff --git a/fluxer_unfurl/src/external_fetch.rs b/fluxer_unfurl/src/external_fetch.rs
new file mode 100644
index 000000000..46cac78a6
--- /dev/null
+++ b/fluxer_unfurl/src/external_fetch.rs
@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+use crate::{http_fetch, network_policy};
+use reqwest::Method;
+use reqwest::header::{
+ ACCEPT, ACCEPT_ENCODING, AUTHORIZATION, CONTENT_ENCODING, CONTENT_TYPE, HeaderMap, HeaderName,
+ HeaderValue,
+};
+use serde::{Deserialize, Serialize};
+use std::time::Duration;
+use tokio::time::timeout;
+use url::Url;
+
+const FETCH_TIMEOUT: Duration = Duration::from_secs(4);
+const MAX_RESPONSE_BYTES: usize = 256 * 1024;
+const MAX_REQUEST_BODY_BYTES: usize = 64 * 1024;
+const DPOP: HeaderName = HeaderName::from_static("dpop");
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct FetchedResponse {
+ pub status: u16,
+ pub headers: Vec<(String, String)>,
+ pub body: String,
+}
+
+pub async fn fetch(
+ client: &reqwest::Client,
+ method: &str,
+ url: &str,
+ headers: &[(String, String)],
+ body: Option<String>,
+) -> anyhow::Result<FetchedResponse> {
+ let method = match method {
+ "GET" => Method::GET,
+ "POST" => Method::POST,
+ _ => anyhow::bail!("unsupported method"),
+ };
+ let url = network_policy::parse_url(url)?;
+ validate_target(&url)?;
+ if body
+ .as_ref()
+ .is_some_and(|b| b.len() > MAX_REQUEST_BODY_BYTES)
+ {
+ anyhow::bail!("request body too large");
+ }
+ let headers = forwarded_headers(headers)?;
+ timeout(FETCH_TIMEOUT, async move {
+ network_policy::validate_url(&url).await?;
+ let mut request = client.request(method, url).headers(headers);
+ if let Some(body) = body {
+ request = request.body(body);
+ }
+ let response = request.send().await?;
+ let status = response.status().as_u16();
+ if let Some(encoding) = response.headers().get(CONTENT_ENCODING)
+ && !encoding
+ .to_str()
+ .is_ok_and(|value| value.trim().eq_ignore_ascii_case("identity"))
+ {
+ anyhow::bail!("encoded response bodies are not accepted");
+ }
+ if response
+ .content_length()
+ .is_some_and(|len| len > MAX_RESPONSE_BYTES as u64)
+ {
+ anyhow::bail!("response body too large");
+ }
+ let headers = response
+ .headers()
+ .iter()
+ .filter_map(|(name, value)| {
+ value
+ .to_str()
+ .ok()
+ .map(|value| (name.as_str().to_owned(), value.to_owned()))
+ })
+ .collect();
+ let bytes = http_fetch::read_body(response, MAX_RESPONSE_BYTES).await?;
+ Ok(FetchedResponse {
+ status,
+ headers,
+ body: String::from_utf8(bytes)
+ .map_err(|_| anyhow::anyhow!("response body is not UTF-8"))?,
+ })
+ })
+ .await
+ .map_err(|_| anyhow::anyhow!("request timed out"))?
+}
+
+fn validate_target(url: &Url) -> anyhow::Result<()> {
+ if url.scheme() != "https" || url.port().is_some() || url.fragment().is_some() {
+ anyhow::bail!("blocked URL");
+ }
+ if !matches!(url.host(), Some(url::Host::Domain(_))) {
+ anyhow::bail!("blocked URL");
+ }
+ Ok(())
+}
+
+fn forwarded_headers(headers: &[(String, String)]) -> anyhow::Result<HeaderMap> {
+ let mut forwarded = HeaderMap::new();
+ for (name, value) in headers {
+ let name = HeaderName::from_bytes(name.as_bytes())?;
+ if [ACCEPT, AUTHORIZATION, CONTENT_TYPE, DPOP].contains(&name) {
+ forwarded.append(name, HeaderValue::from_str(value)?);
+ }
+ }
+ forwarded.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity"));
+ Ok(forwarded)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn only_https_hostnames_on_the_default_port() {
+ for raw in [
+ "https://bsky.social/xrpc/x",
+ "https://plc.directory/did:plc:abc",
+ ] {
+ assert!(validate_target(&Url::parse(raw).unwrap()).is_ok(), "{raw}");
+ }
+ for raw in [
+ "http://bsky.social/",
+ "https://bsky.social:8443/",
+ "https://1.1.1.1/",
+ "https://[2606:4700::1111]/",
+ "https://bsky.social/#x",
+ ] {
+ assert!(validate_target(&Url::parse(raw).unwrap()).is_err(), "{raw}");
+ }
+ }
+
+ #[test]
+ fn forwards_only_known_headers_and_asks_for_identity() {
+ let forwarded = forwarded_headers(&[
+ ("Accept".into(), "application/json".into()),
+ ("DPoP".into(), "proof".into()),
+ ("Authorization".into(), "DPoP token".into()),
+ (
+ "Content-Type".into(),
+ "application/x-www-form-urlencoded".into(),
+ ),
+ ("Host".into(), "internal".into()),
+ ("Accept-Encoding".into(), "br".into()),
+ ("Cookie".into(), "a=b".into()),
+ ])
+ .unwrap();
+ assert_eq!(forwarded.len(), 5);
+ assert_eq!(forwarded[ACCEPT_ENCODING], "identity");
+ assert_eq!(forwarded[DPOP], "proof");
+ assert!(forwarded.get("host").is_none());
+ assert!(forwarded.get("cookie").is_none());
+ }
+}
diff --git a/fluxer_unfurl/src/main.rs b/fluxer_unfurl/src/main.rs
index 5804c28ff..7e1e81248 100644
--- a/fluxer_unfurl/src/main.rs
+++ b/fluxer_unfurl/src/main.rs
@@ -7,6 +7,7 @@ mod direct_media;
mod dns_txt;
mod domain_verification;
mod embed_normalizer;
+mod external_fetch;
mod html_markdown;
mod html_parser;
mod http_fetch;
diff --git a/fluxer_unfurl/src/router_impl.rs b/fluxer_unfurl/src/router_impl.rs
index e2080e9f2..54d304f46 100644
--- a/fluxer_unfurl/src/router_impl.rs
+++ b/fluxer_unfurl/src/router_impl.rs
@@ -44,6 +44,8 @@ impl RouterService for UnfurlRouter {
match req {
UnfurlRequest::Unfurl { url, .. } => url.clone(),
UnfurlRequest::VerifyDomain { domain, .. } => domain.clone(),
+ UnfurlRequest::Fetch { url, .. } => url.clone(),
+ UnfurlRequest::ResolveTxt { name } => name.clone(),
}
}
@@ -70,7 +72,9 @@ impl RouterService for UnfurlRouter {
)
))
}
- UnfurlRequest::VerifyDomain { .. } => None,
+ UnfurlRequest::VerifyDomain { .. }
+ | UnfurlRequest::Fetch { .. }
+ | UnfurlRequest::ResolveTxt { .. } => None,
}
}
@@ -95,7 +99,9 @@ impl RouterService for UnfurlRouter {
self.l1.get(&key)
}
}
- UnfurlRequest::VerifyDomain { .. } => None,
+ UnfurlRequest::VerifyDomain { .. }
+ | UnfurlRequest::Fetch { .. }
+ | UnfurlRequest::ResolveTxt { .. } => None,
}
}
@@ -122,7 +128,9 @@ impl RouterService for UnfurlRouter {
);
}
}
- UnfurlRequest::VerifyDomain { .. } => {}
+ UnfurlRequest::VerifyDomain { .. }
+ | UnfurlRequest::Fetch { .. }
+ | UnfurlRequest::ResolveTxt { .. } => {}
}
}
}
diff --git a/fluxer_unfurl/src/shard_impl.rs b/fluxer_unfurl/src/shard_impl.rs
index 54d753c13..86a6e0b15 100644
--- a/fluxer_unfurl/src/shard_impl.rs
+++ b/fluxer_unfurl/src/shard_impl.rs
@@ -211,6 +211,10 @@ fn base_http_client_builder() -> reqwest::ClientBuilder {
.user_agent("Mozilla/5.0 (compatible; Fluxerbot/1.0; +https://fluxer.app)")
.timeout(Duration::from_secs(10))
.redirect(reqwest::redirect::Policy::none())
+ .no_gzip()
+ .no_brotli()
+ .no_deflate()
+ .no_zstd()
}
fn external_http_client() -> reqwest::Client {
@@ -294,6 +298,32 @@ impl ShardService for UnfurlShard {
}),
}
}
+ UnfurlRequest::Fetch {
+ method,
+ url,
+ headers,
+ body,
+ } => {
+ match crate::external_fetch::fetch(&self.http_client, &method, &url, &headers, body)
+ .await
+ {
+ Ok(response) => Ok(UnfurlResponse::Fetched(response)),
+ Err(err) => Ok(UnfurlResponse::Failed {
+ message: err.to_string(),
+ }),
+ }
+ }
+ UnfurlRequest::ResolveTxt { name } => {
+ if let Err(err) = crate::dns_txt::validate_record_name(&name) {
+ return Ok(UnfurlResponse::Failed {
+ message: err.to_string(),
+ });
+ }
+ let records = crate::dns_txt::first_answer(&name, |records| !records.is_empty())
+ .await
+ .unwrap_or_default();
+ Ok(UnfurlResponse::Txt(records))
+ }
}
}
}
diff --git a/fluxer_unfurl/src/types.rs b/fluxer_unfurl/src/types.rs
index cd388eb65..70c5ed823 100644
--- a/fluxer_unfurl/src/types.rs
+++ b/fluxer_unfurl/src/types.rs
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
+use crate::external_fetch::FetchedResponse;
use serde::{Deserialize, Serialize};
use std::sync::Arc;
@@ -22,6 +23,17 @@ pub enum UnfurlRequest {
domain: String,
token: String,
},
+ Fetch {
+ method: String,
+ url: String,
+ #[serde(default)]
+ headers: Vec<(String, String)>,
+ #[serde(default)]
+ body: Option<String>,
+ },
+ ResolveTxt {
+ name: String,
+ },
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
@@ -37,6 +49,8 @@ pub enum NsfwMode {
pub enum UnfurlResponse {
Resolved(Arc<UnfurlResult>),
Verified(bool),
+ Fetched(FetchedResponse),
+ Txt(Vec<String>),
Failed { message: String },
}
@@ -209,7 +223,7 @@ mod tests {
assert!(youtube_api_key.is_none());
assert!(klipy_api_key.is_none());
}
- UnfurlRequest::VerifyDomain { .. } => panic!("expected an unfurl request"),
+ _ => panic!("expected an unfurl request"),
}
}
}