A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
feat(unfurl): sign ActivityPub fetches for rich embeds (#3651)
Hampusad0b0bbdcc5b
diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yamlindex cd69cabe5..4c20d9e69 100644--- a/.github/workflows/helm.yaml+++ b/.github/workflows/helm.yaml@@ -138,6 +138,7 @@ jobs:expect_failure "letters and digits only" --set $d --set-string 'secrets.values.FLUXER_VALKEY_PASSWORD=0123456789abcdef0123456789abcd:@'expect_failure "worker.replicas must be 1" --set $d --set worker.replicas=2expect_failure "at least 32 bytes" --set $d --set secrets.values.FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64=c2hvcnQ=+ expect_failure "DER RSA private key" --set $d --set secrets.values.FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64=c2hvcnQ=expect_failure "set postgres.external.host" --set $d --set postgres.enabled=falseexpect_failure "set together" --set $d --set secrets.values.FLUXER_VAPID_PUBLIC_KEY=abcexpect_failure "is a secret and would be stored in plain text" --set $d --set env.FLUXER_EMAIL_SMTP_PASSWORD=not-a-real-passworddiff --git a/Cargo.lock b/Cargo.lockindex 6a1a60a69..314e7d79b 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1904,6 +1904,7 @@ version = "0.1.0"dependencies = ["ammonia","anyhow",+ "base64 0.23.1","chrono","encoding_rs","entities",@@ -1914,6 +1915,7 @@ dependencies = ["moka","regex","reqwest",+ "ring","scraper","serde","serde_json",diff --git a/deploy/helm/fluxer/files/Caddyfile b/deploy/helm/fluxer/files/Caddyfileindex ea06f1326..3fe3623cd 100644--- a/deploy/helm/fluxer/files/Caddyfile+++ b/deploy/helm/fluxer/files/Caddyfile@@ -68,6 +68,10 @@import upstream api:8080}+ handle /.well-known/webfinger {+ import upstream api:8080+ }+handle {import upstream app-proxy:8080}diff --git a/deploy/helm/fluxer/templates/NOTES.txt b/deploy/helm/fluxer/templates/NOTES.txtindex 724467bbd..e1ddbef36 100644--- a/deploy/helm/fluxer/templates/NOTES.txt+++ b/deploy/helm/fluxer/templates/NOTES.txt@@ -50,7 +50,7 @@ Voice media does not pass through the edge. Open these ports to the internet:{{- end }}{{- end }}-Back up the Secret {{ include "fluxer.secretName" . }}{{ if $v.postgres.enabled }}, the database in postgres-0{{ end }}{{ if $v.seaweedfs.enabled }} and the uploads on the volume data-seaweedfs-0{{ end }}.+Back up the {{ if $v.secrets.existingSecret }}Secret {{ $v.secrets.existingSecret }}{{ else }}Secrets fluxer-env and fluxer-unfurl-env{{ end }}{{ if $v.postgres.enabled }}, the database in postgres-0{{ end }}{{ if $v.seaweedfs.enabled }} and the uploads on the volume data-seaweedfs-0{{ end }}.{{- if $v.secrets.existingSecret }}After editing the Secret {{ $v.secrets.existingSecret }}, restart the release:diff --git a/deploy/helm/fluxer/templates/_helpers.tpl b/deploy/helm/fluxer/templates/_helpers.tplindex 267977434..7acfc6635 100644--- a/deploy/helm/fluxer/templates/_helpers.tpl+++ b/deploy/helm/fluxer/templates/_helpers.tpl@@ -185,7 +185,7 @@ public: {{ substr 56 121 $der | b64enc | replace "+" "-" | replace "/" "_" | tri{{- else if and (hasPrefix "FLUXER_LIVEKIT_" $k) (eq (include "fluxer.livekitMode" $) "off") }}{{- else if and (eq $k "FLUXER_VALKEY_PASSWORD") (not $.Values.valkey.enabled) }}{{- else if and (eq $k "FLUXER_NATS_AUTH_TOKEN") (not $.Values.nats.enabled) }}-{{- else if and (eq $k "FLUXER_SETUP_TOKEN") (or $.Values.secrets.existingSecret (and (not $.Values.secrets.generate) (not (include "fluxer.string" (get $.Values.secrets.values $k | default ""))))) }}+{{- else if and (has $k (list "FLUXER_SETUP_TOKEN" "FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64")) (or $.Values.secrets.existingSecret (and (not $.Values.secrets.generate) (not (include "fluxer.string" (get $.Values.secrets.values $k | default ""))))) }}{{- else }}{{- $keys = append $keys $k }}{{- end }}@@ -204,6 +204,10 @@ public: {{ substr 56 121 $der | b64enc | replace "+" "-" | replace "/" "_" | tri{{- if $v.secrets.existingSecret }}{{- $out = $existing }}{{- else }}+{{- $unfurlFound := lookup "v1" "Secret" .Release.Namespace "fluxer-unfurl-env" }}+{{- range $k, $val := ($unfurlFound.data | default dict) }}+{{- $_ := set $existing $k (b64dec $val) }}+{{- end }}{{- $noGenerate := include "fluxer.noGenerateKeys" . | fromJsonArray }}{{- $explicit := $v.secrets.values | default dict }}{{- $vapidPublic := include "fluxer.string" (get $explicit "FLUXER_VAPID_PUBLIC_KEY" | default "") }}@@ -240,6 +244,8 @@ public: {{ substr 56 121 $der | b64enc | replace "+" "-" | replace "/" "_" | tri{{- else if $v.secrets.generate }}{{- if eq $k "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64" }}{{- $_ := set $out $k (randBytes 32) }}+{{- else if eq $k "FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64" }}+{{- $_ := set $out $k (genPrivateKey "rsa" | replace "-----BEGIN RSA PRIVATE KEY-----" "" | replace "-----END RSA PRIVATE KEY-----" "" | nospace) }}{{- else }}{{- $_ := set $out $k (randAlphaNum 64) }}{{- end }}diff --git a/deploy/helm/fluxer/templates/_validate.tpl b/deploy/helm/fluxer/templates/_validate.tplindex 5e30259a1..9c4728978 100644--- a/deploy/helm/fluxer/templates/_validate.tpl+++ b/deploy/helm/fluxer/templates/_validate.tpl@@ -149,6 +149,11 @@{{- fail "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 must be standard base64 of at least 32 bytes, for example the output of openssl rand -base64 32" }}{{- end }}{{- end }}+{{- with get $data "FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64" }}+{{- if or (not (regexMatch "^[A-Za-z0-9+/]+={0,2}$" .)) (lt (len (b64dec .)) 256) }}+{{- fail "FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64 must be the standard base64 of a DER RSA private key, for example the output of openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -outform DER | base64 | tr -d '\\n'" }}+{{- end }}+{{- end }}{{- $long := list "FLUXER_SUDO_MODE_SECRET" "FLUXER_CONNECTION_INITIATION_SECRET" "FLUXER_PROFILE_PSEUDONYM_SECRET" "FLUXER_GATEWAY_RPC_AUTH_TOKEN" "FLUXER_ERLANG_COOKIE" "FLUXER_MEDIA_PROXY_SECRET_KEY" "FLUXER_ADMIN_SECRET_KEY_BASE" "FLUXER_ADMIN_OAUTH_CLIENT_SECRET" "FLUXER_SETUP_TOKEN" }}{{- if ne (include "fluxer.livekitMode" .) "off" }}{{- $long = append $long "FLUXER_LIVEKIT_API_SECRET" }}diff --git a/deploy/helm/fluxer/templates/secret.yaml b/deploy/helm/fluxer/templates/secret.yamlindex dbdad05b7..885acbbc4 100644--- a/deploy/helm/fluxer/templates/secret.yaml+++ b/deploy/helm/fluxer/templates/secret.yaml@@ -11,7 +11,22 @@ metadata:helm.sh/resource-policy: keeptype: Opaquedata:- {{- range $k, $val := include "fluxer.secretData" . | fromJson }}+ {{- range $k, $val := omit (include "fluxer.secretData" . | fromJson) "FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64" }}{{ $k }}: {{ $val | b64enc | quote }}{{- end }}+---+apiVersion: v1+kind: Secret+metadata:+ name: fluxer-unfurl-env+ namespace: {{ .Release.Namespace }}+ labels:+ {{- include "fluxer.labels" (dict "root" . "name" "fluxer-unfurl-env" "component" "config") | nindent 4 }}+ annotations:+ helm.sh/resource-policy: keep+type: Opaque+{{- with get (include "fluxer.secretData" . | fromJson) "FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64" }}+data:+ FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64: {{ . | b64enc | quote }}+{{- end }}{{- end }}diff --git a/deploy/helm/fluxer/templates/svc.yaml b/deploy/helm/fluxer/templates/svc.yamlindex b3e31b656..38a2e2690 100644--- a/deploy/helm/fluxer/templates/svc.yaml+++ b/deploy/helm/fluxer/templates/svc.yaml@@ -18,6 +18,9 @@{{- end }}{{- if eq $name "unfurl" }}{{- $shardEnv = append $shardEnv (dict "name" "FLUXER_MEDIA_PROXY_ENDPOINT" "value" "http://media-proxy:8080") }}+{{- if and (not $.Values.secrets.existingSecret) (get (include "fluxer.secretData" $ | fromJson) "FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64") }}+{{- $shardEnv = append $shardEnv (dict "name" "FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64" "valueFrom" (dict "secretKeyRef" (dict "name" "fluxer-unfurl-env" "key" "FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64"))) }}+{{- end }}{{- end }}{{- include "fluxer.app" (dict "root" $ "name" $name "component" "svc" "w" $router "image" (printf "fluxer-%s" $name) "port" 8090 "containerName" "router""env" (append $common (dict "name" "FLUXER_SVC_MODE" "value" "router"))diff --git a/deploy/helm/fluxer/values.schema.json b/deploy/helm/fluxer/values.schema.jsonindex 8ed6b3a97..d9cd3c5be 100644--- a/deploy/helm/fluxer/values.schema.json+++ b/deploy/helm/fluxer/values.schema.json@@ -296,6 +296,9 @@"FLUXER_SETUP_TOKEN": {"type": ["string", "null"]},+ "FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64": {+ "type": ["string", "null"]+ },"FLUXER_LIVEKIT_API_KEY": {"type": ["string", "null"]},diff --git a/deploy/helm/fluxer/values.yaml b/deploy/helm/fluxer/values.yamlindex 623c5aa74..9029deea2 100644--- a/deploy/helm/fluxer/values.yaml+++ b/deploy/helm/fluxer/values.yaml@@ -106,6 +106,7 @@ secrets:# existingSecret, or with generate false and no value here, the API makes# one on its first start and writes it to its log once.FLUXER_SETUP_TOKEN: ""+ FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64: ""# LiveKit API key.FLUXER_LIVEKIT_API_KEY: fluxer# LiveKit API secret, at least 32 characters.diff --git a/fluxer_api/src/api/activity_pub/ActivityPub.ts b/fluxer_api/src/api/activity_pub/ActivityPub.tsnew file mode 100644index 000000000..2bdab67ff--- /dev/null+++ b/fluxer_api/src/api/activity_pub/ActivityPub.ts@@ -0,0 +1,69 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import {createPublicKey} from 'node:crypto';++const ACTIVITY_STREAMS_CONTEXT = 'https://www.w3.org/ns/activitystreams';+const SECURITY_CONTEXT = 'https://w3id.org/security/v1';++export const ACTIVITY_PUB_CONTENT_TYPE = 'application/activity+json';+export const WEBFINGER_CONTENT_TYPE = 'application/jrd+json';++function activityPubActorUrl(apiPublic: string): string {+ return `${apiPublic.replace(/\/+$/u, '')}/activitypub/actor`;+}++export function activityPubKeyId(apiPublic: string): string {+ return `${activityPubActorUrl(apiPublic)}#main-key`;+}++export function activityPubPublicKeyPem(pkcs1Base64: string): string {+ return createPublicKey({key: Buffer.from(pkcs1Base64, 'base64'), format: 'der', type: 'pkcs1'})+ .export({type: 'spki', format: 'pem'})+ .toString();+}++function actorAcct(actorUrl: string): string {+ const url = new URL(actorUrl);+ return `${url.hostname}@${url.host}`;+}++export function buildActivityPubActor(apiPublic: string, publicKeyPem: string) {+ const actorUrl = activityPubActorUrl(apiPublic);+ return {+ '@context': [ACTIVITY_STREAMS_CONTEXT, SECURITY_CONTEXT],+ id: actorUrl,+ type: 'Application',+ preferredUsername: new URL(actorUrl).hostname,+ inbox: `${actorUrl}/inbox`,+ outbox: `${actorUrl}/outbox`,+ manuallyApprovesFollowers: true,+ publicKey: {+ id: activityPubKeyId(apiPublic),+ owner: actorUrl,+ publicKeyPem,+ },+ };+}++export function buildActivityPubOutbox(apiPublic: string) {+ return {+ '@context': ACTIVITY_STREAMS_CONTEXT,+ id: `${activityPubActorUrl(apiPublic)}/outbox`,+ type: 'OrderedCollection',+ totalItems: 0,+ orderedItems: [],+ };+}++export function buildWebFinger(apiPublic: string, resource: string | undefined) {+ const actorUrl = activityPubActorUrl(apiPublic);+ const acct = actorAcct(actorUrl);+ const requested = resource?.trim() ?? '';+ if (requested.toLowerCase() !== `acct:${acct}`.toLowerCase() && requested !== actorUrl) {+ return null;+ }+ return {+ subject: `acct:${acct}`,+ links: [{rel: 'self', type: ACTIVITY_PUB_CONTENT_TYPE, href: actorUrl}],+ };+}diff --git a/fluxer_api/src/api/activity_pub/ActivityPubController.ts b/fluxer_api/src/api/activity_pub/ActivityPubController.tsnew file mode 100644index 000000000..8808851d8--- /dev/null+++ b/fluxer_api/src/api/activity_pub/ActivityPubController.ts@@ -0,0 +1,57 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import {+ ACTIVITY_PUB_CONTENT_TYPE,+ buildActivityPubActor,+ buildActivityPubOutbox,+ buildWebFinger,+ WEBFINGER_CONTENT_TYPE,+} from '@app/api/activity_pub/ActivityPub';+import {Config} from '@app/api/Config';+import {RateLimitMiddleware} from '@app/api/middleware/RateLimitMiddleware';+import {getUnfurlerService} from '@app/api/middleware/ServiceSingletons';+import {RateLimitConfigs} from '@app/api/RateLimitConfig';+import type {HonoApp} from '@app/api/types/HonoEnv';+import {AppNotFoundHandler} from '@fluxer/errors/src/domains/core/ErrorHandlers';++const DOCUMENT_CACHE_CONTROL = 'public, max-age=300';++export function ActivityPubController(app: HonoApp) {+ app.get('/activitypub/actor', RateLimitMiddleware(RateLimitConfigs.ACTIVITY_PUB_DOCUMENT), async (ctx) => {+ const publicKeyPem = await getUnfurlerService().activityPubPublicKey();+ if (publicKeyPem === null) {+ return AppNotFoundHandler(ctx);+ }+ return ctx.body(JSON.stringify(buildActivityPubActor(Config.endpoints.apiPublic, publicKeyPem)), 200, {+ 'Content-Type': ACTIVITY_PUB_CONTENT_TYPE,+ 'Cache-Control': DOCUMENT_CACHE_CONTROL,+ });+ });+ app.get('/activitypub/actor/outbox', RateLimitMiddleware(RateLimitConfigs.ACTIVITY_PUB_DOCUMENT), async (ctx) => {+ if ((await getUnfurlerService().activityPubPublicKey()) === null) {+ return AppNotFoundHandler(ctx);+ }+ return ctx.body(JSON.stringify(buildActivityPubOutbox(Config.endpoints.apiPublic)), 200, {+ 'Content-Type': ACTIVITY_PUB_CONTENT_TYPE,+ });+ });+ app.post('/activitypub/actor/inbox', RateLimitMiddleware(RateLimitConfigs.ACTIVITY_PUB_DOCUMENT), async (ctx) => {+ if ((await getUnfurlerService().activityPubPublicKey()) === null) {+ return AppNotFoundHandler(ctx);+ }+ return ctx.body(null, 202);+ });+ app.get('/.well-known/webfinger', RateLimitMiddleware(RateLimitConfigs.ACTIVITY_PUB_DOCUMENT), async (ctx) => {+ if ((await getUnfurlerService().activityPubPublicKey()) === null) {+ return AppNotFoundHandler(ctx);+ }+ const document = buildWebFinger(Config.endpoints.apiPublic, ctx.req.query('resource'));+ if (document === null) {+ return AppNotFoundHandler(ctx);+ }+ return ctx.body(JSON.stringify(document), 200, {+ 'Content-Type': WEBFINGER_CONTENT_TYPE,+ 'Cache-Control': DOCUMENT_CACHE_CONTROL,+ });+ });+}diff --git a/fluxer_api/src/api/app/ControllerRegistry.ts b/fluxer_api/src/api/app/ControllerRegistry.tsindex 3f025454b..a01559756 100644--- a/fluxer_api/src/api/app/ControllerRegistry.ts+++ b/fluxer_api/src/api/app/ControllerRegistry.ts@@ -1,5 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later+import {ActivityPubController} from '@app/api/activity_pub/ActivityPubController';import {AdminDownloadController} from '@app/api/admin/AdminDownloadController';import {registerAdminControllers} from '@app/api/admin/controllers/index';import {AttachmentController} from '@app/api/attachment/AttachmentController';@@ -58,6 +59,7 @@ export function registerControllers(routes: HonoApp, config: APIConfig): void {ConnectionController(routes);BlueskyOAuthController(routes);InstanceController(routes);+ ActivityPubController(routes);OpenAPIController(routes);DownloadController(routes);ExperimentController(routes);diff --git a/fluxer_api/src/api/infrastructure/IUnfurlerService.ts b/fluxer_api/src/api/infrastructure/IUnfurlerService.tsindex 6a001da88..84bea19a2 100644--- a/fluxer_api/src/api/infrastructure/IUnfurlerService.ts+++ b/fluxer_api/src/api/infrastructure/IUnfurlerService.ts@@ -47,4 +47,6 @@ export abstract class IUnfurlerService {abstract fetchExternal(request: ExternalFetchRequest): Promise<ExternalFetchResponse>;abstract resolveTxt(name: string): Promise<Array<string>>;++ abstract activityPubPublicKey(): Promise<string | null>;}diff --git a/fluxer_api/src/api/infrastructure/NatsUnfurlerService.ts b/fluxer_api/src/api/infrastructure/NatsUnfurlerService.tsindex 3fd7da392..bdb38e12d 100644--- a/fluxer_api/src/api/infrastructure/NatsUnfurlerService.ts+++ b/fluxer_api/src/api/infrastructure/NatsUnfurlerService.ts@@ -1,5 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later+import {activityPubPublicKeyPem} from '@app/api/activity_pub/ActivityPub';import type {MediaProxyNsfwMode} from '@app/api/infrastructure/IMediaService';import {type ExternalFetchRequest,@@ -25,6 +26,8 @@ const NATS_UNFURL_CACHE_ONLY_TIMEOUT_MS = 1000;const NATS_VERIFY_DOMAIN_TIMEOUT_MS = 8000;const NATS_FETCH_TIMEOUT_MS = 6000;const NATS_RESOLVE_TXT_TIMEOUT_MS = 4000;+const NATS_ACTIVITY_PUB_KEY_TIMEOUT_MS = 2000;+const ACTIVITY_PUB_KEY_CACHE_TTL_MS = 5 * 60 * 1000;interface NatsUnfurlRequest {op: 'Unfurl';@@ -34,6 +37,7 @@ interface NatsUnfurlRequest {cache_only: boolean;youtube_api_key: string | null;klipy_api_key: string | null;+ activity_pub_key_id: string | null;}interface NatsUnfurlInnerResult {@@ -124,6 +128,16 @@ function isNatsTxtResponse(value: unknown): value is NatsTxtResponse {return false;}+type NatsActivityPubKeyResponse = {ActivityPubKey: string | null};++function isNatsActivityPubKeyResponse(value: unknown): value is NatsActivityPubKeyResponse {+ return (+ isJsonRecord(value) &&+ 'ActivityPubKey' in value &&+ (value.ActivityPubKey === null || typeof value.ActivityPubKey === 'string')+ );+}+function mapUnfurlTransportError(error: unknown): Error {if (error instanceof FluxerError) {return error;@@ -139,11 +153,13 @@ function mapUnfurlTransportError(error: unknown): Error {export class NatsUnfurlerService extends IUnfurlerService {private readonly connectionManager: INatsConnectionManager;+ private activityPubKeyCache: {pem: string | null; expiresAt: number} | null = null;constructor(connectionManager: INatsConnectionManager,private readonly resolveYoutubeApiKey: (() => Promise<string | null>) | null = null,private readonly resolveKlipyApiKey: (() => Promise<string | null>) | null = null,+ private readonly activityPubKeyId: string | null = null,) {super();this.connectionManager = connectionManager;@@ -163,6 +179,7 @@ export class NatsUnfurlerService extends IUnfurlerService {cache_only: options.cacheOnly === true,youtube_api_key: this.resolveYoutubeApiKey ? await this.resolveYoutubeApiKey() : null,klipy_api_key: this.resolveKlipyApiKey ? await this.resolveKlipyApiKey() : null,+ activity_pub_key_id: this.activityPubKeyId,};const response = await requestSvcJson(this.connectionManager, NATS_UNFURL_SUBJECT, request, {timeoutMs: options.cacheOnly === true ? NATS_UNFURL_CACHE_ONLY_TIMEOUT_MS : NATS_UNFURL_TIMEOUT_MS,@@ -249,4 +266,28 @@ export class NatsUnfurlerService extends IUnfurlerService {}throw new BadGatewayError({message: `[nats-unfurl] TXT lookup failed: ${response.Failed.message}`});}++ async activityPubPublicKey(): Promise<string | null> {+ const cached = this.activityPubKeyCache;+ if (cached !== null && cached.expiresAt > Date.now()) {+ return cached.pem;+ }+ const response = await requestSvcJson(+ this.connectionManager,+ NATS_UNFURL_SUBJECT,+ {op: 'ActivityPubKey'},+ {+ timeoutMs: NATS_ACTIVITY_PUB_KEY_TIMEOUT_MS,+ guard: isNatsActivityPubKeyResponse,+ service: 'nats-unfurl',+ invalidPayloadError: () => new BadGatewayError({message: '[nats-unfurl] invalid response payload'}),+ },+ ).catch((error: unknown) => {+ Logger.warn({error}, '[nats-unfurl] failed to fetch ActivityPub public key');+ throw new ServiceUnavailableError({message: '[nats-unfurl] ActivityPub public key is unavailable'});+ });+ const pem = response.ActivityPubKey === null ? null : activityPubPublicKeyPem(response.ActivityPubKey);+ this.activityPubKeyCache = {pem, expiresAt: Date.now() + ACTIVITY_PUB_KEY_CACHE_TTL_MS};+ return pem;+ }}diff --git a/fluxer_api/src/api/middleware/ServiceSingletons.ts b/fluxer_api/src/api/middleware/ServiceSingletons.tsindex 44c077b83..2680dce08 100644--- a/fluxer_api/src/api/middleware/ServiceSingletons.ts+++ b/fluxer_api/src/api/middleware/ServiceSingletons.ts@@ -1,5 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later+import {activityPubKeyId} from '@app/api/activity_pub/ActivityPub';import {AdminRepository} from '@app/api/admin/AdminRepository';import {AdminApiKeyRepository} from '@app/api/admin/repositories/AdminApiKeyRepository';import {AdminArchiveRepository} from '@app/api/admin/repositories/AdminArchiveRepository';@@ -349,6 +350,7 @@ const getDefaultUnfurlerService = singleton(() => {manager,async () => instanceConfigRepository.getEffectiveYoutubeApiKey(),async () => (await instanceConfigRepository.getEffectiveGifConfig()).klipy_api_key,+ activityPubKeyId(Config.endpoints.apiPublic),);});diff --git a/fluxer_api/src/api/rate_limit_configs/MiscRateLimitConfig.ts b/fluxer_api/src/api/rate_limit_configs/MiscRateLimitConfig.tsindex f18cdf9d3..143904069 100644--- a/fluxer_api/src/api/rate_limit_configs/MiscRateLimitConfig.ts+++ b/fluxer_api/src/api/rate_limit_configs/MiscRateLimitConfig.ts@@ -8,6 +8,10 @@ export const MiscRateLimitConfigs = {bucket: 'instance:info',config: {limit: 60, windowMs: ms('1 minute')},} as RouteRateLimitConfig,+ ACTIVITY_PUB_DOCUMENT: {+ bucket: 'activity_pub:document',+ config: {limit: 60, windowMs: ms('1 minute')},+ } as RouteRateLimitConfig,INSTANCE_SETUP_ACCOUNT_IDENTITY: {bucket: 'instance:setup:account_identity',config: {limit: 10, windowMs: ms('1 minute')},diff --git a/fluxer_api/src/api/test/mocks/NoopUnfurlerService.ts b/fluxer_api/src/api/test/mocks/NoopUnfurlerService.tsindex f2e7722ee..aac36c791 100644--- a/fluxer_api/src/api/test/mocks/NoopUnfurlerService.ts+++ b/fluxer_api/src/api/test/mocks/NoopUnfurlerService.ts@@ -29,4 +29,8 @@ export class NoopUnfurlerService extends IUnfurlerService {override async resolveTxt(_name: string): Promise<Array<string>> {return [];}++ override async activityPubPublicKey(): Promise<string | null> {+ return null;+ }}diff --git a/fluxer_docs/src/content/docs/operator/docker-compose.mdx b/fluxer_docs/src/content/docs/operator/docker-compose.mdxindex 33d9bda2a..789636e3a 100644--- a/fluxer_docs/src/content/docs/operator/docker-compose.mdx+++ b/fluxer_docs/src/content/docs/operator/docker-compose.mdx@@ -33,6 +33,7 @@ for key in POSTGRES_PASSWORD MEILI_MASTER_KEY FLUXER_S3_SECRET_KEY FLUXER_SUDO_Mdonesed -i "s|^FLUXER_SETUP_TOKEN=.*|FLUXER_SETUP_TOKEN=$(openssl rand -hex 16)|" .envsed -i "s|^FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64=.*|FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64=$(openssl rand -base64 32)|" .env+sed -i "s|^FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64=.*|FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64=$(openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -outform DER | base64 | tr -d '\n')|" .envopenssl ecparam -name prime256v1 -genkey -noout -out vapid.pemsed -i "s|^FLUXER_VAPID_PUBLIC_KEY=.*|FLUXER_VAPID_PUBLIC_KEY=$(openssl ec -in vapid.pem -pubout -outform DER 2>/dev/null | tail -c 65 | base64 | tr '/+' '_-' | tr -d '=\n')|" .envsed -i "s|^FLUXER_VAPID_PRIVATE_KEY=.*|FLUXER_VAPID_PRIVATE_KEY=$(openssl ec -in vapid.pem -outform DER 2>/dev/null | tail -c +8 | head -c 32 | base64 | tr '/+' '_-' | tr -d '=\n')|" .envdiff --git a/fluxer_docs/src/content/docs/operator/kubernetes.mdx b/fluxer_docs/src/content/docs/operator/kubernetes.mdxindex 2fafc2c11..1de1cc102 100644--- a/fluxer_docs/src/content/docs/operator/kubernetes.mdx+++ b/fluxer_docs/src/content/docs/operator/kubernetes.mdx@@ -102,19 +102,19 @@ helm upgrade fluxer ./fluxer/deploy/helm/fluxer -n fluxer -f fluxer-values.yaml## Back up and restore-Back up the Secret and the database.+Back up the Secrets and the database.```sh-kubectl -n fluxer get secret fluxer-env -o yaml \- | grep -v -E '^ (resourceVersion|uid|creationTimestamp):' > fluxer-env.yaml+kubectl -n fluxer get secret fluxer-env fluxer-unfurl-env -o yaml \+ | grep -v -E '^ (resourceVersion|uid|creationTimestamp):' > fluxer-env.yamlkubectl -n fluxer exec postgres-0 -- sh -c 'pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom' > fluxer.dump```-A restored database only works with the same Secret. Keep `fluxer-env.yaml` as safe as the dump.+A restored database only works with the same Secrets. Keep `fluxer-env.yaml` as safe as the dump.Uploads are on the volume `data-seaweedfs-0`. Back it up with a snapshot from your storage provider.-To restore, put the Secret back before you install, and the new release uses the same secrets.+To restore, put the Secrets back before you install, and the new release uses the same secrets.```shkubectl create namespace fluxer@@ -142,10 +142,10 @@ helm upgrade fluxer ./fluxer/deploy/helm/fluxer -n fluxer -f fluxer-values.yamlhelm uninstall fluxer -n fluxer```-The Secret and the volumes stay, and a new install with the same release name and namespace uses them again. To delete everything, delete them too.+The Secrets and the volumes stay, and a new install with the same release name and namespace uses them again. To delete everything, delete them too.```sh-kubectl -n fluxer delete secret fluxer-env+kubectl -n fluxer delete secret fluxer-env fluxer-unfurl-envkubectl -n fluxer delete pvc -l app.kubernetes.io/instance=fluxerkubectl delete namespace fluxer```diff --git a/fluxer_rexctl/internal/instance/bundleenv.go b/fluxer_rexctl/internal/instance/bundleenv.goindex a62fc11b7..9b442c028 100644--- a/fluxer_rexctl/internal/instance/bundleenv.go+++ b/fluxer_rexctl/internal/instance/bundleenv.go@@ -268,6 +268,11 @@ func secretHelp(g *secretGroup) (string, []string) {commands = []string{"openssl rand -hex 32"}case secrets.KindBase64_32:commands = []string{"openssl rand -base64 32"}+ case secrets.KindRSA:+ commands = []string{+ "openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -outform DER |",+ " base64 | tr -d '\\n'",+ }case secrets.KindAlnum64:commands = []string{"openssl rand -base64 96 | tr -dc A-Za-z0-9 | head -c 64"}case secrets.KindVAPID:diff --git a/fluxer_rexctl/internal/instance/shipped.json b/fluxer_rexctl/internal/instance/shipped.jsonindex f38931bb9..30344c146 100644--- a/fluxer_rexctl/internal/instance/shipped.json+++ b/fluxer_rexctl/internal/instance/shipped.json@@ -40,6 +40,7 @@"73faca6872d91a8f24a71efde33307aef8a3559241851bdc703ce0c4b3ce80ab","75eaee5650ddd1c9af5d414440be8f7736df86ee4556827620a681627a9d1fc8","762fdb27709acc69b7f9f6032476818641a5aeda6516481e5c93080cff3419cb",+ "769e0f3e2b80bfee2789606a0430a9a08f58d070b2e1452f3d67e01cdddd9479","78740c67acc541e8ae2432defbf3df21c548a895e2aee070052541d0fdbe533a","78c81c9bc7b1715189783331824a60fcda365a26a52f3aaeb7209913735a9261","7a020c862ad7a0d49664afc66c90b8f77e33fa4fd0282704fc8805b955e45b65",@@ -96,6 +97,7 @@"6f88e16d9fa2d2c57431304bcdb1384236ad0f82f3e205692e10050c57381352","734b9210e29c837d2312a7a03b19dc5aa7233848399debab9a5e474fa7fe1570","7fd66a72a740258ed71fcaebce927bdc9b2528177cfeb5ca9e0c0ee778dd8769",+ "8b1e007032b4c5fd2e36f3bfa9435609cb626b0aec10c3e07dc8bee6887ccc33","cb68f62d48bc54bc908840c6dce85843ac5580e6196e5a9b978dd14ffb3d5280","e9ab1a7ad99211172f68cfa2728beeb86dd4ed220635cab5f8461bb2b49cb023","f02d23b1c650c157ab53ee7cc2057c0d6dec7b93f5dbd99982c594cdd6c766d1"@@ -183,6 +185,7 @@"893b3367ed49800478b7152c8f46190be2ad9196a04bab99cb53a964242daed4","8b89586104a5a1f3dd6d997ce43ff935d08298983660b6fec9f8b21739dada1a","8f0ae94476bbf3aabae086154b1ffa23ce77427ee445f48043e11cc96860faef",+ "8fc464560ddfd9068035acafde0dbe4fa96aad78b43e382ccd03ab89f1369cfa","90fcefc650f64640eb0527ec287b721748635e9214902c076d2b7abf3faa624c","9140244e91fb0262f6e6c53a9f7e89780213018f7edbc1c9e6925a143e9cee06","9341b083d98c743766dfa13a1424f9a138c5ba9f2ff03a89caf7590685fa22a0",diff --git a/fluxer_rexctl/internal/secrets/secrets.go b/fluxer_rexctl/internal/secrets/secrets.goindex 99f744805..1e5ed006f 100644--- a/fluxer_rexctl/internal/secrets/secrets.go+++ b/fluxer_rexctl/internal/secrets/secrets.go@@ -4,6 +4,8 @@ package secretsimport ("crypto/rand"+ "crypto/rsa"+ "crypto/x509""encoding/base64""encoding/hex""errors"@@ -19,6 +21,7 @@ const (KindBase64_32KindVAPIDKindFixed+ KindRSA)const Unset = "CHANGE_ME"@@ -46,6 +49,7 @@ var Required = []Spec{{Key: "FLUXER_ADMIN_OAUTH_CLIENT_SECRET", Kind: KindHex32},{Key: "LIVEKIT_API_SECRET", Kind: KindHex32, Stateful: true},{Key: "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64", Kind: KindBase64_32},+ {Key: "FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64", Kind: KindRSA},{Key: VAPIDPublicKey, Kind: KindVAPID, Stateful: true},{Key: VAPIDPrivateKey, Kind: KindVAPID, Stateful: true},}@@ -100,6 +104,18 @@ func Base64(nBytes int) (string, error) {return base64.StdEncoding.EncodeToString(b), nil}+func RSAKey() (string, error) {+ key, err := rsa.GenerateKey(rand.Reader, 2048)+ if err != nil {+ return "", err+ }+ der, err := x509.MarshalPKCS8PrivateKey(key)+ if err != nil {+ return "", err+ }+ return base64.StdEncoding.EncodeToString(der), nil+}+func IsUnset(value string) bool {v := strings.TrimSpace(value)return v == "" || v == Unset@@ -113,6 +129,8 @@ func Generate(s Spec) (string, error) {return Alnum(64)case KindBase64_32:return Base64(32)+ case KindRSA:+ return RSAKey()case KindFixed:return s.Fixed, nil}diff --git a/fluxer_rexctl/internal/stack/files/Caddyfile b/fluxer_rexctl/internal/stack/files/Caddyfileindex ea06f1326..3fe3623cd 100644--- a/fluxer_rexctl/internal/stack/files/Caddyfile+++ b/fluxer_rexctl/internal/stack/files/Caddyfile@@ -68,6 +68,10 @@import upstream api:8080}+ handle /.well-known/webfinger {+ import upstream api:8080+ }+handle {import upstream app-proxy:8080}diff --git a/fluxer_rexctl/internal/stack/files/docker-compose.yml b/fluxer_rexctl/internal/stack/files/docker-compose.ymlindex 6e9ad5032..be0d0f913 100644--- a/fluxer_rexctl/internal/stack/files/docker-compose.yml+++ b/fluxer_rexctl/internal/stack/files/docker-compose.yml@@ -851,6 +851,7 @@ services:FLUXER_SVC_NAME: unfurlFLUXER_SVC_MODE: combinedFLUXER_MEDIA_PROXY_ENDPOINT: http://media-proxy:8080+ FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64: ${FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64:-}healthcheck: *fluxer-svc-healthcheckdepends_on:nats: {condition: service_healthy}diff --git a/fluxer_unfurl/Cargo.toml b/fluxer_unfurl/Cargo.tomlindex 375d3ce35..3da8fe5ef 100644--- a/fluxer_unfurl/Cargo.toml+++ b/fluxer_unfurl/Cargo.toml@@ -9,6 +9,7 @@ publish = falsefluxer_common = { path = "../fluxer_common" }ammonia = "4.2.3"anyhow = "1.0.104"+base64 = "0.23.1"chrono = { version = "0.4.45", default-features = false, features = ["std"] }encoding_rs = "0.8.41"entities = "1.0.1"@@ -18,6 +19,7 @@ infer = "0.22.0"moka = { version = "0.12.16", features = ["future", "sync"] }regex = "1.13"reqwest = { version = "0.13.5", default-features = false, features = ["json", "rustls", "stream"] }+ring = "0.17.14"scraper = "0.27"serde = { version = "1.0.229", features = ["derive", "rc"] }serde_json = "1.0.151"diff --git a/fluxer_unfurl/src/activity_pub/fetcher.rs b/fluxer_unfurl/src/activity_pub/fetcher.rsindex cee3f6a1d..0815e8d8a 100644--- a/fluxer_unfurl/src/activity_pub/fetcher.rs+++ b/fluxer_unfurl/src/activity_pub/fetcher.rs@@ -1,12 +1,13 @@// SPDX-License-Identifier: AGPL-3.0-or-lateruse super::limits;+use super::signature::RequestSigner;use super::types::{ActivityPubActor, ActivityPubPost, InstanceInfo, MastodonInstance, MastodonPost,};use crate::http_fetch;use reqwest::header::{ACCEPT, HeaderMap, HeaderValue};-use std::time::Duration;+use std::time::{Duration, SystemTime};use tracing::debug;use url::Url;@@ -17,18 +18,39 @@ const ACCEPT_ACTIVITY_PUB: &str = concat!("profile=\"https://www.w3.org/ns/activitystreams\"");-pub async fn fetch_activity_pub(client: &reqwest::Client, url: &str) -> Option<ActivityPubPost> {- let mut headers = HeaderMap::new();- headers.insert(ACCEPT, HeaderValue::from_static(ACCEPT_ACTIVITY_PUB));- let result = http_fetch::fetch_url_with_headers(- client,+#[derive(Clone, Copy)]+pub struct ActivityPubClient<'a> {+ pub http: &'a reqwest::Client,+ pub signer: Option<&'a RequestSigner>,+}++async fn fetch_activity_pub_object(+ client: ActivityPubClient<'_>,+ url: &str,+) -> Option<http_fetch::FetchResult> {+ http_fetch::fetch_url_with_headers(+ client.http,url,- headers,+ |hop| {+ let mut headers = match client.signer {+ Some(signer) => signer.headers(hop, SystemTime::now())?,+ None => HeaderMap::new(),+ };+ headers.insert(ACCEPT, HeaderValue::from_static(ACCEPT_ACTIVITY_PUB));+ Ok(headers)+ },limits::OBJECT_MAX_BYTES,Duration::from_secs(5),).await- .ok()?;+ .ok()+}++pub async fn fetch_activity_pub(+ client: ActivityPubClient<'_>,+ url: &str,+) -> Option<ActivityPubPost> {+ let result = fetch_activity_pub_object(client, url).await?;if !(200..=299).contains(&result.status) {debug!(url, status = result.status, "ActivityPub fetch failed");@@ -46,20 +68,10 @@ pub async fn fetch_activity_pub(client: &reqwest::Client, url: &str) -> Option<A}pub async fn fetch_activity_pub_actor(- client: &reqwest::Client,+ client: ActivityPubClient<'_>,url: &str,) -> Option<ActivityPubActor> {- let mut headers = HeaderMap::new();- headers.insert(ACCEPT, HeaderValue::from_static(ACCEPT_ACTIVITY_PUB));- let result = http_fetch::fetch_url_with_headers(- client,- url,- headers,- limits::OBJECT_MAX_BYTES,- Duration::from_secs(5),- )- .await- .ok()?;+ let result = fetch_activity_pub_object(client, url).await?;if !(200..=299).contains(&result.status) {return None;diff --git a/fluxer_unfurl/src/activity_pub/mod.rs b/fluxer_unfurl/src/activity_pub/mod.rsindex 439c7eb79..97aa65fb6 100644--- a/fluxer_unfurl/src/activity_pub/mod.rs+++ b/fluxer_unfurl/src/activity_pub/mod.rs@@ -3,23 +3,42 @@pub mod fetcher;pub mod formatter;pub mod limits;+pub mod signature;pub mod types;pub mod utils;+pub mod visibility;use crate::html_parser;use crate::media_proxy::MediaProxyClient;use crate::types::MessageEmbed;use crate::types::NsfwMode;+use fetcher::ActivityPubClient;use url::Url;pub async fn try_resolve(- client: &reqwest::Client,+ client: ActivityPubClient<'_>,media_proxy: &MediaProxyClient,nsfw_mode: NsfwMode,url: &Url,html: &str,) -> Option<Vec<MessageEmbed>> {- let ap_link = html_parser::find_activity_pub_link(html)?;+ let Some(ap_link) = html_parser::find_activity_pub_link(html) else {+ if client.signer.is_some()+ && utils::extract_post_id(url).is_some()+ && html_parser::has_fediverse_profile_username(html)+ {+ return resolve_activity_pub_post(+ client,+ media_proxy,+ nsfw_mode,+ url,+ html,+ url.as_str(),+ )+ .await;+ }+ return None;+ };let ap_url = utils::resolve_relative_url(url, &ap_link)?;@@ -27,38 +46,17 @@ pub async fn try_resolve(return None;}- let ap_post = fetcher::fetch_activity_pub(client, &ap_url).await;- if let Some(post) = ap_post {- let mut context = build_context(client, url, Some(html)).await;- if let Some(parent_url) = post.in_reply_to.as_deref() {- context.in_reply_to = fetch_activity_pub_reply_context(client, parent_url).await;- }- let author_actor = fetch_post_author_actor(client, &post).await;- let quote_child =- fetch_activity_pub_quote_child(client, media_proxy, nsfw_mode, &post).await;- let embeds = formatter::format_activity_pub_post(- &post,- url,- &context,- media_proxy,- nsfw_mode,- formatter::ActivityPubFormatOptions {- author_actor: author_actor.as_ref(),- quote_child,- is_nested: false,- },- )- .await;- if !embeds.is_empty() {- return Some(embeds);- }+ if let Some(embeds) =+ resolve_activity_pub_post(client, media_proxy, nsfw_mode, url, html, &ap_url).await+ {+ return Some(embeds);}let post_id = utils::extract_post_id(url)?;- let mastodon_post = fetcher::fetch_mastodon_status(client, url, &post_id).await?;+ let mastodon_post = fetcher::fetch_mastodon_status(client.http, url, &post_id).await?;let mut context = build_context(client, url, Some(html)).await;if mastodon_post.in_reply_to_id.is_some() && mastodon_post.in_reply_to_account_id.is_some() {- context.in_reply_to = fetch_mastodon_reply_context(client, url, &mastodon_post).await;+ context.in_reply_to = fetch_mastodon_reply_context(client.http, url, &mastodon_post).await;}let embeds =formatter::format_mastodon_post(&mastodon_post, url, &context, media_proxy, nsfw_mode)@@ -71,12 +69,50 @@ pub async fn try_resolve(}}-async fn build_context(- client: &reqwest::Client,+async fn resolve_activity_pub_post(+ client: ActivityPubClient<'_>,+ media_proxy: &MediaProxyClient,+ nsfw_mode: NsfwMode,+ url: &Url,+ html: &str,+ ap_url: &str,+) -> Option<Vec<MessageEmbed>> {+ let post = fetcher::fetch_activity_pub(client, ap_url).await?;+ let author_actor = fetch_post_author_actor(client, &post).await;+ if visibility::is_hidden_from_unauthed_web(&post, author_actor.as_ref()) {+ return None;+ }+ let mut context = build_context(client, url, Some(html)).await;+ if let Some(parent_url) = post.in_reply_to.as_deref() {+ context.in_reply_to = fetch_activity_pub_reply_context(client, parent_url).await;+ }+ let quote_child = fetch_activity_pub_quote_child(client, media_proxy, nsfw_mode, &post).await;+ let embeds = formatter::format_activity_pub_post(+ &post,+ url,+ &context,+ media_proxy,+ nsfw_mode,+ formatter::ActivityPubFormatOptions {+ author_actor: author_actor.as_ref(),+ quote_child,+ is_nested: false,+ },+ )+ .await;+ if embeds.is_empty() {+ None+ } else {+ Some(embeds)+ }+}++pub(crate) async fn build_context(+ client: ActivityPubClient<'_>,url: &Url,html: Option<&str>,) -> types::ActivityPubContext {- let instance = fetcher::fetch_instance_info(client, url).await;+ let instance = fetcher::fetch_instance_info(client.http, url).await;let apple_touch_icon = html.and_then(|html| html_parser::find_apple_touch_icon(html, url));let clean_host = url.host_str()@@ -100,8 +136,8 @@ async fn build_context(}}-async fn fetch_post_author_actor(- client: &reqwest::Client,+pub(crate) async fn fetch_post_author_actor(+ client: ActivityPubClient<'_>,post: &types::ActivityPubPost,) -> Option<types::ActivityPubActor> {let actor_url = post.attributed_to.as_ref()?.as_str()?;@@ -109,11 +145,14 @@ async fn fetch_post_author_actor(}async fn fetch_activity_pub_reply_context(- client: &reqwest::Client,+ client: ActivityPubClient<'_>,parent_url: &str,) -> Option<types::ActivityPubReplyContext> {let parent = fetcher::fetch_activity_pub(client, parent_url).await?;let author_actor = fetch_post_author_actor(client, &parent).await;+ if visibility::is_hidden_from_unauthed_web(&parent, author_actor.as_ref()) {+ return None;+ }let author = match author_actor.as_ref() {Some(actor) => activity_pub_handle_from_actor(actor, parent_url),None => parent@@ -148,7 +187,7 @@ async fn fetch_mastodon_reply_context(}async fn fetch_activity_pub_quote_child(- client: &reqwest::Client,+ client: ActivityPubClient<'_>,media_proxy: &MediaProxyClient,nsfw_mode: NsfwMode,post: &types::ActivityPubPost,@@ -159,6 +198,10 @@ async fn fetch_activity_pub_quote_child(.or(post.quote_uri.as_deref()).or(post.misskey_quote.as_deref())?;let quote = fetcher::fetch_activity_pub(client, quote_url).await?;+ let author_actor = fetch_post_author_actor(client, "e).await;+ if visibility::is_hidden_from_unauthed_web("e, author_actor.as_ref()) {+ return None;+ }let fallback = Url::parse(quote_url).ok()?;let quote_url = quote.url@@ -169,7 +212,6 @@ async fn fetch_activity_pub_quote_child(if let Some(parent_url) = quote.in_reply_to.as_deref() {quote_context.in_reply_to = fetch_activity_pub_reply_context(client, parent_url).await;}- let author_actor = fetch_post_author_actor(client, "e).await;let embeds = formatter::format_activity_pub_post("e,"e_url,diff --git a/fluxer_unfurl/src/activity_pub/signature.rs b/fluxer_unfurl/src/activity_pub/signature.rsnew file mode 100644index 000000000..54ee97756--- /dev/null+++ b/fluxer_unfurl/src/activity_pub/signature.rs@@ -0,0 +1,209 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++use base64::prelude::*;+use chrono::{DateTime, Utc};+use reqwest::header::{DATE, HOST, HeaderMap, HeaderName, HeaderValue};+use ring::rand::SystemRandom;+use ring::signature::{RSA_PKCS1_SHA256, RsaKeyPair};+use std::sync::Arc;+use std::time::SystemTime;+use url::Url;++const SIGNATURE: HeaderName = HeaderName::from_static("signature");++pub fn parse_signing_key(encoded: &str) -> anyhow::Result<RsaKeyPair> {+ let compact: String = encoded+ .chars()+ .filter(|c| !c.is_ascii_whitespace())+ .collect();+ let der = BASE64_STANDARD.decode(compact)?;+ RsaKeyPair::from_pkcs8(&der)+ .or_else(|_| RsaKeyPair::from_der(&der))+ .map_err(|err| anyhow::anyhow!("invalid RSA private key: {err}"))+}++pub fn public_key_base64(key: &RsaKeyPair) -> String {+ BASE64_STANDARD.encode(key.public().as_ref())+}++#[derive(Clone)]+pub struct RequestSigner {+ key: Arc<RsaKeyPair>,+ key_id: String,+}++impl RequestSigner {+ pub fn new(key: Arc<RsaKeyPair>, key_id: String) -> Self {+ Self { key, key_id }+ }++ pub fn headers(&self, url: &Url, at: SystemTime) -> anyhow::Result<HeaderMap> {+ let host_name = url+ .host_str()+ .ok_or_else(|| anyhow::anyhow!("signed request URL has no host"))?;+ let host = match url.port() {+ Some(port) => format!("{host_name}:{port}"),+ None => host_name.to_owned(),+ };+ let date = DateTime::<Utc>::from(at)+ .format("%a, %d %b %Y %H:%M:%S GMT")+ .to_string();+ let target = match url.query() {+ Some(query) => format!("{}?{query}", url.path()),+ None => url.path().to_owned(),+ };+ let signing_string = format!("(request-target): get {target}\nhost: {host}\ndate: {date}");+ let mut signature = vec![0; self.key.public().modulus_len()];+ self.key+ .sign(+ &RSA_PKCS1_SHA256,+ &SystemRandom::new(),+ signing_string.as_bytes(),+ &mut signature,+ )+ .map_err(|_| anyhow::anyhow!("failed to sign ActivityPub request"))?;+ let signature_header = format!(+ "keyId=\"{}\",algorithm=\"rsa-sha256\",headers=\"(request-target) host date\",signature=\"{}\"",+ self.key_id,+ BASE64_STANDARD.encode(signature)+ );++ let mut headers = HeaderMap::new();+ headers.insert(HOST, HeaderValue::from_str(&host)?);+ headers.insert(DATE, HeaderValue::from_str(&date)?);+ headers.insert(SIGNATURE, HeaderValue::from_str(&signature_header)?);+ Ok(headers)+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::http_fetch::{DEFAULT_MAX_BYTES, fetch_url_with_headers};+ use serde::Deserialize;+ use std::io::{BufRead, BufReader, Write};+ use std::net::TcpListener;+ use std::time::{Duration, UNIX_EPOCH};++ #[derive(Deserialize)]+ struct Vectors {+ key_id: String,+ private_key_pkcs8_base64: String,+ private_key_pkcs1_base64: String,+ public_key_pkcs1_base64: String,+ vectors: Vec<Vector>,+ }++ #[derive(Deserialize)]+ struct Vector {+ name: String,+ url: String,+ unix_seconds: u64,+ host: String,+ date: String,+ signature: String,+ }++ #[test]+ fn signs_known_answer_vectors_with_pkcs8_and_pkcs1_keys() {+ let raw = include_str!("testdata/http_signature_vectors.json");+ let vectors: Vectors = serde_json::from_str(raw).expect("valid vector file");+ for encoded in [+ &vectors.private_key_pkcs8_base64,+ &vectors.private_key_pkcs1_base64,+ ] {+ let key = parse_signing_key(encoded).expect("test key parses");+ assert_eq!(public_key_base64(&key), vectors.public_key_pkcs1_base64);+ let signer = RequestSigner::new(Arc::new(key), vectors.key_id.clone());+ for vector in &vectors.vectors {+ let url = Url::parse(&vector.url).unwrap();+ let at = UNIX_EPOCH + Duration::from_secs(vector.unix_seconds);+ let headers = signer.headers(&url, at).unwrap();+ assert_eq!(headers[HOST], vector.host.as_str(), "{}", vector.name);+ assert_eq!(headers[DATE], vector.date.as_str(), "{}", vector.name);+ assert_eq!(+ headers[SIGNATURE],+ vector.signature.as_str(),+ "{}",+ vector.name+ );+ }+ }+ }++ #[tokio::test]+ async fn signs_every_redirect_hop_for_its_own_target_and_host() {+ let raw = include_str!("testdata/http_signature_vectors.json");+ let vectors: Vectors = serde_json::from_str(raw).expect("valid vector file");+ let key = parse_signing_key(&vectors.private_key_pkcs8_base64).expect("test key parses");+ let signer = RequestSigner::new(Arc::new(key), vectors.key_id.clone());+ let at = UNIX_EPOCH + Duration::from_secs(1_790_000_000);++ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ let proxy = format!("http://{}", listener.local_addr().unwrap());+ let server = std::thread::spawn(move || {+ let mut hops = Vec::new();+ for response in [+ "HTTP/1.1 301 Moved Permanently\r\nLocation: http://8.8.8.8:8080/users/alice/statuses/1\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",+ "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}",+ ] {+ let (mut stream, _) = listener.accept().unwrap();+ let mut reader = BufReader::new(stream.try_clone().unwrap());+ let mut request = Vec::new();+ loop {+ let mut line = String::new();+ reader.read_line(&mut line).unwrap();+ if line.trim_end().is_empty() {+ break;+ }+ request.push(line.trim_end().to_owned());+ }+ stream.write_all(response.as_bytes()).unwrap();+ hops.push(request);+ }+ hops+ });++ let client = reqwest::Client::builder()+ .redirect(reqwest::redirect::Policy::none())+ .proxy(reqwest::Proxy::http(proxy).unwrap())+ .build()+ .unwrap();+ let result = fetch_url_with_headers(+ &client,+ "http://1.1.1.1/@alice/1?x=1",+ |url| signer.headers(url, at),+ DEFAULT_MAX_BYTES,+ Duration::from_secs(5),+ )+ .await+ .unwrap();+ assert_eq!(result.status, 200);++ let hops = server.join().unwrap();+ for (request, url) in hops.iter().zip([+ "http://1.1.1.1/@alice/1?x=1",+ "http://8.8.8.8:8080/users/alice/statuses/1",+ ]) {+ let expected = signer.headers(&Url::parse(url).unwrap(), at).unwrap();+ let header = |name: &str| {+ request+ .iter()+ .find_map(|line| {+ let (key, value) = line.split_once(':')?;+ key.eq_ignore_ascii_case(name)+ .then(|| value.trim().to_owned())+ })+ .unwrap_or_default()+ };+ assert_eq!(request[0], format!("GET {url} HTTP/1.1"));+ assert_eq!(header("host"), expected[HOST].to_str().unwrap(), "{url}");+ assert_eq!(+ header("signature"),+ expected[SIGNATURE].to_str().unwrap(),+ "{url}"+ );+ }+ assert_ne!(hops[0], hops[1]);+ }+}diff --git a/fluxer_unfurl/src/activity_pub/testdata/http_signature_vectors.json b/fluxer_unfurl/src/activity_pub/testdata/http_signature_vectors.jsonnew file mode 100644index 000000000..ad4056321--- /dev/null+++ b/fluxer_unfurl/src/activity_pub/testdata/http_signature_vectors.json@@ -0,0 +1,26 @@+{+ "key_id": "https://fluxer.example/api/activitypub/actor#main-key",+ "private_key_pkcs8_base64": "MIIEuwIBADANBgkqhkiG9w0BAQEFAASCBKUwggShAgEAAoIBAQC0MyYJ7+OeIyy9ExVonsw4OdiDKaH85o0UQ/ie1AcriECTr8xbubziM/V6OBJ5csOhgUFogghVy1RR9jtdszG1uxmLS2Q32xmgfRQvWK8YhETd3WBP41N7MaPLAm4i/fBtPT6ZVqV67S8MDQhlwQzhSBg6erk2f+zOBK9XdOAfZbjoOfxe0aJwN3NCBEYB5oO8oWVfmMw0a8FSHWsaXMv2Ctfbv9OxHjlJ3J9Ojk5a7rOnfnHcolu2rVvdasibcmmBRVR2ZUfbGL5MlO+I4opItnetuVPKB+/pMt8h4tddqafM8lovvPUE2UDC4WEcA7EBpeKD6md9+auZyR7DkB8vAgMBAAECgf8BLc1rbaDaMprTOJ5o4bJbNgWr9vy1X3mGh289FtkRbkVr0um6UzxdVgyIny7P22rbvlZC1/0G2uks0MkbSAwhMmBpfFII+j0lsDxKTrBPgJqXpSs5Y//P3jl0fxHuAWmiipNYxWwtLeDaPlMkjK2zTGWlNmi5Hofu3a4u3HDDw89gbtJBatScrQoy/krSR8IoZ7KDvA7l4eULwjtbnBRYvQJKP28IlI0QE+0QdC4eaS/F58LCf5ask2Poj1bpfq7OZVyuq/as0SLbWskt2kVduuZrUpDzklH+Y1hU9eswcXW90F4EVSiPK3KZnVhhjg4E3TzufUq4QAKyXrTb4UECgYEA2NQaNBPsc7eDGtzocK79A/36tkt6dowGfx/dg57YeEUNel7U13WFe8Dtke+CM94qpw0u4dg9RWYVBU3/cIXEsmIk1aFqnbKuzCxkr4o/9RbjKW0NbkdRwcpa0r2hSTMHdUAfutPGbtvIwHMAhF1MJGPjTT8g/Jv5ykf1Yfw5lSECgYEA1MEJ9Ro7bcGQeYwcq0/KEpPYBICENLvKVocbEa399vf5pvyXjbgMwXPUoHDxDusZlnaxEfrJoex2HW61+W7DTvYD9icGl9h/TE8TZ2EHhKMhtRY1J3jnmsRjkThZJL3dcyjS4nj0GfboD1c8roPmZahFBv/L1MK+NWhQte4t2k8CgYA57oGJxID6cd9/f+JCqR5EwMkJh6USo1rhADyI/BnOGHT/avLnFikx5VwUD6/z97nhpC4QfVvxfNLE9g/3mmXxWzOIOHZp5+G3/2xvUjrAWvsaiF58RmXbpxsZDz1Wd2HYu+c+XXK5D9PVR2St+NdtKZsDeohvtaTJnOw838zqwQKBgFWThCbTs13VgwBxDEMYKAbK8xP0gbkYZC3j0E6PNt01J3U0mt1FT9QsfDVA2KWQiU1NVrINtzbrfyMdmhLHq8XCDRvg3LUNPz75e6uUisqeN3u4gA7tgfNt1ULfcHPn7wBYRmMUFdOvze4FwAHOSklPotxK3bkp0Ap5eCWFtWnfAoGBAK6iurVoq6O3rVEGVmrLs5Q7USQQmpM8bCxt16JLQF2Gai3JDuTvnRuGEg2nSGb2zTlX/6uzbmr1uSaDcNIX+fb3Vs5nt5PDxHsqAw4C5hNWtx/kBRoCurD+WfTUgpVo/c13KJ+kgXNoFlDhvAlELT68rP8cvaHicJM7J9aJXW0j",+ "private_key_pkcs1_base64": "MIIEoQIBAAKCAQEAtDMmCe/jniMsvRMVaJ7MODnYgymh/OaNFEP4ntQHK4hAk6/MW7m84jP1ejgSeXLDoYFBaIIIVctUUfY7XbMxtbsZi0tkN9sZoH0UL1ivGIRE3d1gT+NTezGjywJuIv3wbT0+mValeu0vDA0IZcEM4UgYOnq5Nn/szgSvV3TgH2W46Dn8XtGicDdzQgRGAeaDvKFlX5jMNGvBUh1rGlzL9grX27/TsR45SdyfTo5OWu6zp35x3KJbtq1b3WrIm3JpgUVUdmVH2xi+TJTviOKKSLZ3rblTygfv6TLfIeLXXamnzPJaL7z1BNlAwuFhHAOxAaXig+pnffmrmckew5AfLwIDAQABAoH/AS3Na22g2jKa0zieaOGyWzYFq/b8tV95hodvPRbZEW5Fa9LpulM8XVYMiJ8uz9tq275WQtf9BtrpLNDJG0gMITJgaXxSCPo9JbA8Sk6wT4Cal6UrOWP/z945dH8R7gFpooqTWMVsLS3g2j5TJIyts0xlpTZouR6H7t2uLtxww8PPYG7SQWrUnK0KMv5K0kfCKGeyg7wO5eHlC8I7W5wUWL0CSj9vCJSNEBPtEHQuHmkvxefCwn+WrJNj6I9W6X6uzmVcrqv2rNEi21rJLdpFXbrma1KQ85JR/mNYVPXrMHF1vdBeBFUojytymZ1YYY4OBN087n1KuEACsl602+FBAoGBANjUGjQT7HO3gxrc6HCu/QP9+rZLenaMBn8f3YOe2HhFDXpe1Nd1hXvA7ZHvgjPeKqcNLuHYPUVmFQVN/3CFxLJiJNWhap2yrswsZK+KP/UW4yltDW5HUcHKWtK9oUkzB3VAH7rTxm7byMBzAIRdTCRj400/IPyb+cpH9WH8OZUhAoGBANTBCfUaO23BkHmMHKtPyhKT2ASAhDS7ylaHGxGt/fb3+ab8l424DMFz1KBw8Q7rGZZ2sRH6yaHsdh1utfluw072A/YnBpfYf0xPE2dhB4SjIbUWNSd455rEY5E4WSS93XMo0uJ49Bn26A9XPK6D5mWoRQb/y9TCvjVoULXuLdpPAoGAOe6BicSA+nHff3/iQqkeRMDJCYelEqNa4QA8iPwZzhh0/2ry5xYpMeVcFA+v8/e54aQuEH1b8XzSxPYP95pl8VsziDh2aefht/9sb1I6wFr7GohefEZl26cbGQ89Vndh2LvnPl1yuQ/T1UdkrfjXbSmbA3qIb7WkyZzsPN/M6sECgYBVk4Qm07Nd1YMAcQxDGCgGyvMT9IG5GGQt49BOjzbdNSd1NJrdRU/ULHw1QNilkIlNTVayDbc2638jHZoSx6vFwg0b4Ny1DT8++XurlIrKnjd7uIAO7YHzbdVC33Bz5+8AWEZjFBXTr83uBcABzkpJT6LcSt25KdAKeXglhbVp3wKBgQCuorq1aKujt61RBlZqy7OUO1EkEJqTPGwsbdeiS0BdhmotyQ7k750bhhINp0hm9s05V/+rs25q9bkmg3DSF/n291bOZ7eTw8R7KgMOAuYTVrcf5AUaArqw/ln01IKVaP3NdyifpIFzaBZQ4bwJRC0+vKz/HL2h4nCTOyfWiV1tIw==",+ "public_key_pkcs1_base64": "MIIBCgKCAQEAtDMmCe/jniMsvRMVaJ7MODnYgymh/OaNFEP4ntQHK4hAk6/MW7m84jP1ejgSeXLDoYFBaIIIVctUUfY7XbMxtbsZi0tkN9sZoH0UL1ivGIRE3d1gT+NTezGjywJuIv3wbT0+mValeu0vDA0IZcEM4UgYOnq5Nn/szgSvV3TgH2W46Dn8XtGicDdzQgRGAeaDvKFlX5jMNGvBUh1rGlzL9grX27/TsR45SdyfTo5OWu6zp35x3KJbtq1b3WrIm3JpgUVUdmVH2xi+TJTviOKKSLZ3rblTygfv6TLfIeLXXamnzPJaL7z1BNlAwuFhHAOxAaXig+pnffmrmckew5AfLwIDAQAB",+ "vectors": [+ {+ "name": "query",+ "url": "https://social.example/@alice/statuses/01M5TJBQKR286WRDHACHHKWFVT?page=2",+ "unix_seconds": 1793000000,+ "host": "social.example",+ "date": "Mon, 26 Oct 2026 07:33:20 GMT",+ "signing_string": "(request-target): get /@alice/statuses/01M5TJBQKR286WRDHACHHKWFVT?page=2\nhost: social.example\ndate: Mon, 26 Oct 2026 07:33:20 GMT",+ "signature": "keyId=\"https://fluxer.example/api/activitypub/actor#main-key\",algorithm=\"rsa-sha256\",headers=\"(request-target) host date\",signature=\"eKTxgVb8fmeSmtNEkAHnQYbw+tOTc9WfA53Q5WPSRw0xyMJ75vSkBEVbwzWojzKzG+bkHyB81TMhcQ3SJimPnll+wYVIj4PiKHcY5o1AAi6aWCFWgJJImKmjTHpG10RUpq9/uyc8coW/1im7zmjKm2WLD1H9udgOgSrZtf3fRKFq23QskW3itu4aaYPjRV1GeJW77O1YsXy7ukfuEbsAD1Hnxzc5GWU7lj4wgUqBqxOxsRAs0tdpsKVli4XdrI8UNDSltHCWuS+Abl0aZG2ZeWqAqUSNFdz201nj/wyE/PmJ10/XNCk3BceM7zCrL47NXuIxg5Fdbasfu+xhUTto3A==\""+ },+ {+ "name": "port",+ "url": "https://b.example:8443/users/bob",+ "unix_seconds": 1793086399,+ "host": "b.example:8443",+ "date": "Tue, 27 Oct 2026 07:33:19 GMT",+ "signing_string": "(request-target): get /users/bob\nhost: b.example:8443\ndate: Tue, 27 Oct 2026 07:33:19 GMT",+ "signature": "keyId=\"https://fluxer.example/api/activitypub/actor#main-key\",algorithm=\"rsa-sha256\",headers=\"(request-target) host date\",signature=\"IRJ7aFcP0TAXVqufJoA0nksS0XF55ZLbLixS1ptoSuJ5gbFS3/FMvQoWFcU9bPp7Z4i7kIZrIcjLiVJ6MzfPqlwmmDisw+o5wJSM1u3DKLF/V8wCHQyBNJFisBpxCjcwVcCSamUw159pMp/qJq9kr08qpHKFpnYW3EwfvQl0a+tfI7fCodOMMNpK3OhWOv6IQYswuBeUwgAZBA2aRiyA2s/XCQIeedzrFzMAHHpyuWHDFXfXCxj0Zm7uTii08nV/wYQxBt0onilT3GPyO1EMRHcwMXkDl3InZdJNwx8buCdFA0CeEOuik5bn5PnHA0Kp+a83iBRc+e8ps/Tk/zaYGQ==\""+ }+ ]+}diff --git a/fluxer_unfurl/src/activity_pub/types.rs b/fluxer_unfurl/src/activity_pub/types.rsindex 64bf27b1b..ef2c1854d 100644--- a/fluxer_unfurl/src/activity_pub/types.rs+++ b/fluxer_unfurl/src/activity_pub/types.rs@@ -25,6 +25,8 @@ pub struct ActivityPubPost {pub content: Option<String>,pub summary: Option<String>,pub attachment: Option<Vec<ActivityPubAttachment>>,+ pub to: Option<serde_json::Value>,+ pub cc: Option<serde_json::Value>,pub in_reply_to: Option<String>,pub likes: Option<ActivityPubCollectionCount>,pub shares: Option<ActivityPubCollectionCount>,@@ -66,6 +68,8 @@ pub struct ActivityPubActor {pub preferred_username: Option<String>,pub url: Option<String>,pub icon: Option<ActivityPubIcon>,+ pub hides_to_public_from_unauthed_web: Option<bool>,+ pub hides_cc_public_from_unauthed_web: Option<bool>,}#[derive(Debug, Clone, Deserialize)]diff --git a/fluxer_unfurl/src/activity_pub/visibility.rs b/fluxer_unfurl/src/activity_pub/visibility.rsnew file mode 100644index 000000000..b9b45ab68--- /dev/null+++ b/fluxer_unfurl/src/activity_pub/visibility.rs@@ -0,0 +1,163 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++use super::types::{ActivityPubActor, ActivityPubPost};++const PUBLIC_URIS: [&str; 3] = [+ "https://www.w3.org/ns/activitystreams#Public",+ "as:Public",+ "Public",+];++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum WebVisibility {+ Public,+ Unlocked,+ Restricted,+}++pub fn is_hidden_from_unauthed_web(+ post: &ActivityPubPost,+ author: Option<&ActivityPubActor>,+) -> bool {+ let Some(author) = author else {+ return false;+ };+ if author.hides_to_public_from_unauthed_web.is_none()+ && author.hides_cc_public_from_unauthed_web.is_none()+ {+ return false;+ }+ let hides_to = author.hides_to_public_from_unauthed_web.unwrap_or(false);+ let hides_cc = author.hides_cc_public_from_unauthed_web.unwrap_or(true);+ match web_visibility(post) {+ WebVisibility::Public => hides_to,+ WebVisibility::Unlocked => hides_cc,+ WebVisibility::Restricted => true,+ }+}++fn web_visibility(post: &ActivityPubPost) -> WebVisibility {+ if addresses_public(post.to.as_ref()) {+ WebVisibility::Public+ } else if addresses_public(post.cc.as_ref()) {+ WebVisibility::Unlocked+ } else {+ WebVisibility::Restricted+ }+}++fn addresses_public(value: Option<&serde_json::Value>) -> bool {+ match value {+ Some(serde_json::Value::String(s)) => is_public_uri(s),+ Some(serde_json::Value::Array(items)) => items+ .iter()+ .filter_map(|item| item.as_str())+ .any(is_public_uri),+ _ => false,+ }+}++fn is_public_uri(value: &str) -> bool {+ PUBLIC_URIS.contains(&value)+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ fn post(value: serde_json::Value) -> ActivityPubPost {+ serde_json::from_value(value).expect("valid post")+ }++ fn actor(value: serde_json::Value) -> ActivityPubActor {+ serde_json::from_value(value).expect("valid actor")+ }++ const PUBLIC: &str = "https://www.w3.org/ns/activitystreams#Public";++ #[test]+ fn public_post_from_an_unhidden_actor_is_shown() {+ let post = post(json!({+ "to": [PUBLIC],+ "cc": ["https://gts.test/users/alice/followers"],+ }));+ let actor = actor(json!({+ "hidesToPublicFromUnauthedWeb": false,+ "hidesCcPublicFromUnauthedWeb": false,+ }));+ assert!(!is_hidden_from_unauthed_web(&post, Some(&actor)));+ }++ #[test]+ fn public_post_is_hidden_when_actor_hides_public_from_web() {+ let post = post(json!({ "to": [PUBLIC] }));+ let actor = actor(json!({+ "hidesToPublicFromUnauthedWeb": true,+ "hidesCcPublicFromUnauthedWeb": false,+ }));+ assert!(is_hidden_from_unauthed_web(&post, Some(&actor)));+ }++ #[test]+ fn unlisted_post_is_hidden_by_gotosocial_default() {+ let post = post(json!({+ "to": ["https://gts.test/users/alice/followers"],+ "cc": [PUBLIC],+ }));+ let actor = actor(json!({ "hidesToPublicFromUnauthedWeb": false }));+ assert!(is_hidden_from_unauthed_web(&post, Some(&actor)));+ }++ #[test]+ fn unlisted_post_is_shown_when_actor_allows_cc_on_web() {+ let post = post(json!({+ "to": ["https://gts.test/users/alice/followers"],+ "cc": [PUBLIC],+ }));+ let actor = actor(json!({+ "hidesToPublicFromUnauthedWeb": false,+ "hidesCcPublicFromUnauthedWeb": false,+ }));+ assert!(!is_hidden_from_unauthed_web(&post, Some(&actor)));+ }++ #[test]+ fn followers_only_post_is_always_hidden_for_a_gotosocial_actor() {+ let post = post(json!({+ "to": ["https://gts.test/users/alice/followers"],+ }));+ let actor = actor(json!({+ "hidesToPublicFromUnauthedWeb": false,+ "hidesCcPublicFromUnauthedWeb": false,+ }));+ assert!(is_hidden_from_unauthed_web(&post, Some(&actor)));+ }++ #[test]+ fn non_gotosocial_actor_without_flags_is_never_filtered() {+ let post = post(json!({+ "to": ["https://mastodon.test/users/bob/followers"],+ "cc": [PUBLIC],+ }));+ let actor = actor(json!({ "preferredUsername": "bob" }));+ assert!(!is_hidden_from_unauthed_web(&post, Some(&actor)));+ }++ #[test]+ fn missing_author_actor_is_never_filtered() {+ let post = post(json!({ "to": ["https://gts.test/users/alice/followers"] }));+ assert!(!is_hidden_from_unauthed_web(&post, None));+ }++ #[test]+ fn single_string_and_alias_public_addresses_are_recognised() {+ let string_public = post(json!({ "to": PUBLIC }));+ let hides_to = actor(json!({ "hidesToPublicFromUnauthedWeb": true }));+ assert!(is_hidden_from_unauthed_web(&string_public, Some(&hides_to)));++ let aliased = post(json!({ "cc": "as:Public" }));+ let hides_cc = actor(json!({ "hidesCcPublicFromUnauthedWeb": true }));+ assert!(is_hidden_from_unauthed_web(&aliased, Some(&hides_cc)));+ }+}diff --git a/fluxer_unfurl/src/html_parser.rs b/fluxer_unfurl/src/html_parser.rsindex 4cd38cd95..38bd79455 100644--- a/fluxer_unfurl/src/html_parser.rs+++ b/fluxer_unfurl/src/html_parser.rs@@ -216,6 +216,30 @@ pub fn find_activity_pub_link(html: &str) -> Option<String> {None}+pub fn has_fediverse_profile_username(html: &str) -> bool {+ let doc = Html::parse_document(html);+ let meta = MetaTags::parse(&doc);+ meta.all("og:profile:username")+ .chain(meta.all("profile:username"))+ .any(is_acct_shaped)+}++fn is_acct_shaped(value: &str) -> bool {+ let value = value.trim();+ let value = value.strip_prefix('@').unwrap_or(value);+ let Some((local, host)) = value.split_once('@') else {+ return false;+ };+ !local.is_empty()+ && local+ .chars()+ .all(|c| c.is_alphanumeric() || matches!(c, '_' | '.' | '-'))+ && host.contains('.')+ && host.split('.').all(|label| {+ !label.is_empty() && label.chars().all(|c| c.is_alphanumeric() || c == '-')+ })+}+pub fn find_canonical_url(html: &str, base_url: &url::Url) -> Option<String> {let doc = Html::parse_document(html);let sel = Selector::parse("link[rel=\"canonical\"]").ok()?;diff --git a/fluxer_unfurl/src/http_fetch.rs b/fluxer_unfurl/src/http_fetch.rsindex 6dc577d10..54db9bb15 100644--- a/fluxer_unfurl/src/http_fetch.rs+++ b/fluxer_unfurl/src/http_fetch.rs@@ -4,6 +4,7 @@ use crate::network_policy;use reqwest::header::{HeaderMap, LOCATION};use std::time::Duration;use tokio::time::timeout;+use url::Url;pub const DEFAULT_MAX_BYTES: usize = 8 * 1024 * 1024;@@ -38,16 +39,26 @@ pub async fn fetch_url(max_bytes: usize,request_timeout: Duration,) -> anyhow::Result<FetchResult> {- fetch_url_with_headers(client, url, HeaderMap::new(), max_bytes, request_timeout).await+ fetch_url_with_headers(+ client,+ url,+ |_| Ok(HeaderMap::new()),+ max_bytes,+ request_timeout,+ )+ .await}-pub async fn fetch_url_with_headers(+pub async fn fetch_url_with_headers<H>(client: &reqwest::Client,url: &str,- request_headers: HeaderMap,+ request_headers: H,max_bytes: usize,request_timeout: Duration,-) -> anyhow::Result<FetchResult> {+) -> anyhow::Result<FetchResult>+where+ H: Fn(&Url) -> anyhow::Result<HeaderMap>,+{let result = fetch_url_with_headers_maybe_body(client,url,@@ -78,7 +89,7 @@ wherefetch_url_with_headers_maybe_body(client,url,- HeaderMap::new(),+ |_| Ok(HeaderMap::new()),max_bytes,request_timeout,should_read_body,@@ -86,15 +97,16 @@ where.await}-pub async fn fetch_url_with_headers_maybe_body<F>(+pub async fn fetch_url_with_headers_maybe_body<H, F>(client: &reqwest::Client,url: &str,- request_headers: HeaderMap,+ request_headers: H,max_bytes: usize,request_timeout: Duration,should_read_body: F,) -> anyhow::Result<FetchMaybeBodyResult>where+ H: Fn(&Url) -> anyhow::Result<HeaderMap>,F: Fn(&FetchHead) -> bool,{let mut current_url = network_policy::parse_url(url)?;@@ -106,12 +118,10 @@ where}visited.push(current_str);network_policy::validate_url(¤t_url).await?;+ let hop_headers = request_headers(¤t_url)?;let resp = timeout(request_timeout,- client- .get(current_url.clone())- .headers(request_headers.clone())- .send(),+ client.get(current_url.clone()).headers(hop_headers).send(),).await.map_err(|_| anyhow::anyhow!("request timed out"))??;diff --git a/fluxer_unfurl/src/resolvers/default_resolver.rs b/fluxer_unfurl/src/resolvers/default_resolver.rsindex 7bd2903aa..e73da81b0 100644--- a/fluxer_unfurl/src/resolvers/default_resolver.rs+++ b/fluxer_unfurl/src/resolvers/default_resolver.rs@@ -107,10 +107,11 @@ async fn resolve_html(ctx: &ResolveContext<'_>) -> anyhow::Result<ResolverResultself_hosted: ctx.self_hosted,youtube_api_key: ctx.youtube_api_key.clone(),klipy_api_key: ctx.klipy_api_key.clone(),+ activity_pub_signer: ctx.activity_pub_signer.clone(),};if let Some(embeds) = activity_pub::try_resolve(- &ctx.http_client,+ ctx.activity_pub_client(),ctx.media_proxy,ctx.nsfw_mode,&final_url,@@ -495,22 +496,22 @@ async fn try_canonical_activity_pub(if !urls_differ(&canonical_url, final_url) {return None;}- let ap_post =- crate::activity_pub::fetcher::fetch_activity_pub(&ctx.http_client, &canonical).await?;+ let client = ctx.activity_pub_client();+ let ap_post = activity_pub::fetcher::fetch_activity_pub(client, &canonical).await?;let post_url = ap_post.url.as_deref().and_then(|u| Url::parse(u).ok()).unwrap_or_else(|| canonical_url.clone());- let context = build_canonical_ap_context(&ctx.http_client, &post_url).await;- let author_actor = fetch_canonical_author(&ctx.http_client, &ap_post).await;- let embeds = crate::activity_pub::formatter::format_activity_pub_post(+ let context = activity_pub::build_context(client, &post_url, None).await;+ let author_actor = activity_pub::fetch_post_author_actor(client, &ap_post).await;+ let embeds = activity_pub::formatter::format_activity_pub_post(&ap_post,&post_url,&context,ctx.media_proxy,ctx.nsfw_mode,- crate::activity_pub::formatter::ActivityPubFormatOptions {+ activity_pub::formatter::ActivityPubFormatOptions {author_actor: author_actor.as_ref(),quote_child: None,is_nested: false,@@ -524,40 +525,6 @@ async fn try_canonical_activity_pub(}}-async fn build_canonical_ap_context(- client: &reqwest::Client,- url: &Url,-) -> crate::activity_pub::types::ActivityPubContext {- let instance = crate::activity_pub::fetcher::fetch_instance_info(client, url).await;- let clean_host = url- .host_str()- .unwrap_or_default()- .trim_start_matches("www.")- .trim_start_matches("social.")- .trim_start_matches("mstdn.")- .to_owned();- crate::activity_pub::types::ActivityPubContext {- server_domain: instance- .as_ref()- .and_then(|i| i.domain.clone())- .unwrap_or_else(|| clean_host.clone()),- server_title: instance- .as_ref()- .and_then(|i| i.title.clone())- .unwrap_or_else(|| format!("{clean_host} Mastodon")),- server_icon: instance.and_then(|i| i.thumbnail_url),- in_reply_to: None,- }-}--async fn fetch_canonical_author(- client: &reqwest::Client,- post: &crate::activity_pub::types::ActivityPubPost,-) -> Option<crate::activity_pub::types::ActivityPubActor> {- let actor_url = post.attributed_to.as_ref()?.as_str()?;- crate::activity_pub::fetcher::fetch_activity_pub_actor(client, actor_url).await-}-#[cfg(test)]mod tests {use super::*;diff --git a/fluxer_unfurl/src/resolvers/mod.rs b/fluxer_unfurl/src/resolvers/mod.rsindex 9e897006a..fe3fd8ff9 100644--- a/fluxer_unfurl/src/resolvers/mod.rs+++ b/fluxer_unfurl/src/resolvers/mod.rs@@ -12,6 +12,8 @@ pub mod wikipedia;pub mod xkcd;pub mod youtube;+use crate::activity_pub::fetcher::ActivityPubClient;+use crate::activity_pub::signature::RequestSigner;use crate::media_proxy::MediaProxyClient;use crate::types::{MessageEmbed, NsfwMode};use std::future::Future;@@ -27,6 +29,16 @@ pub struct ResolveContext<'mp> {pub self_hosted: bool,pub youtube_api_key: Option<String>,pub klipy_api_key: Option<String>,+ pub activity_pub_signer: Option<RequestSigner>,+}++impl ResolveContext<'_> {+ pub fn activity_pub_client(&self) -> ActivityPubClient<'_> {+ ActivityPubClient {+ http: &self.http_client,+ signer: self.activity_pub_signer.as_ref(),+ }+ }}pub struct ResolverResult {diff --git a/fluxer_unfurl/src/router_impl.rs b/fluxer_unfurl/src/router_impl.rsindex 54d304f46..9ada7dca8 100644--- a/fluxer_unfurl/src/router_impl.rs+++ b/fluxer_unfurl/src/router_impl.rs@@ -46,6 +46,7 @@ impl RouterService for UnfurlRouter {UnfurlRequest::VerifyDomain { domain, .. } => domain.clone(),UnfurlRequest::Fetch { url, .. } => url.clone(),UnfurlRequest::ResolveTxt { name } => name.clone(),+ UnfurlRequest::ActivityPubKey => "activity_pub_key".to_owned(),}}@@ -58,6 +59,7 @@ impl RouterService for UnfurlRouter {cache_only,youtube_api_key,klipy_api_key,+ ..} => {if *bypass_cache {return None;@@ -74,7 +76,8 @@ impl RouterService for UnfurlRouter {}UnfurlRequest::VerifyDomain { .. }| UnfurlRequest::Fetch { .. }- | UnfurlRequest::ResolveTxt { .. } => None,+ | UnfurlRequest::ResolveTxt { .. }+ | UnfurlRequest::ActivityPubKey => None,}}@@ -101,7 +104,8 @@ impl RouterService for UnfurlRouter {}UnfurlRequest::VerifyDomain { .. }| UnfurlRequest::Fetch { .. }- | UnfurlRequest::ResolveTxt { .. } => None,+ | UnfurlRequest::ResolveTxt { .. }+ | UnfurlRequest::ActivityPubKey => None,}}@@ -130,7 +134,8 @@ impl RouterService for UnfurlRouter {}UnfurlRequest::VerifyDomain { .. }| UnfurlRequest::Fetch { .. }- | UnfurlRequest::ResolveTxt { .. } => {}+ | UnfurlRequest::ResolveTxt { .. }+ | UnfurlRequest::ActivityPubKey => {}}}}diff --git a/fluxer_unfurl/src/shard_impl.rs b/fluxer_unfurl/src/shard_impl.rsindex 86a6e0b15..f4e96ac15 100644--- a/fluxer_unfurl/src/shard_impl.rs+++ b/fluxer_unfurl/src/shard_impl.rs@@ -1,5 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later+use crate::activity_pub::signature::{self, RequestSigner};use crate::cache_policy::{UnfurlEntryExpiry, default_empty_entry_lifetime, provider_capability, unfurl_cache_key,};@@ -10,6 +11,7 @@ use crate::types::{NsfwMode, UnfurlRequest, UnfurlResponse, UnfurlResult};use fluxer_svc::config::optional_env;use fluxer_svc::shard::ShardService;use moka::future::Cache;+use ring::signature::RsaKeyPair;use std::sync::Arc;use std::time::Duration;use url::Url;@@ -25,6 +27,7 @@ pub struct UnfurlShard {self_hosted: bool,youtube_api_key: Option<String>,klipy_api_key: Option<String>,+ activity_pub_key: Option<Arc<RsaKeyPair>>,}impl UnfurlShard {@@ -71,6 +74,14 @@ impl UnfurlShard {media_proxy_public_endpoint.as_deref(),media_proxy_http_client,);+ let activity_pub_key =+ optional_env("FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64").map(|encoded| {+ let key = signature::parse_signing_key(&encoded).unwrap_or_else(|err| {+ panic!("invalid FLUXER_ACTIVITYPUB_SIGNING_KEY_BASE64: {err}")+ });+ tracing::info!("ActivityPub request signing enabled");+ Arc::new(key)+ });Self {cache: build_l2_cache(lifetime, default_empty_entry_lifetime()),@@ -83,6 +94,7 @@ impl UnfurlShard {.or_else(|| optional_env("YOUTUBE_API_KEY")),klipy_api_key: optional_env("FLUXER_KLIPY_API_KEY").or_else(|| optional_env("KLIPY_API_KEY")),+ activity_pub_key,}}@@ -92,6 +104,7 @@ impl UnfurlShard {nsfw_mode: NsfwMode,youtube_api_key: Option<&str>,klipy_api_key: Option<&str>,+ activity_pub_key_id: Option<&str>,) -> anyhow::Result<UnfurlResult> {let parsed = Url::parse(url_str)?;@@ -114,6 +127,11 @@ impl UnfurlShard {klipy_api_key: klipy_api_key.map(str::to_owned).or_else(|| self.klipy_api_key.clone()),+ activity_pub_signer: self+ .activity_pub_key+ .clone()+ .zip(activity_pub_key_id)+ .map(|(key, key_id)| RequestSigner::new(key, key_id.to_owned())),};if let Some(idx) = matched_resolver_idx {@@ -249,6 +267,7 @@ impl ShardService for UnfurlShard {cache_only,ref youtube_api_key,ref klipy_api_key,+ ref activity_pub_key_id,} => {let nsfw = nsfw_mode.unwrap_or_default();let cache_key = unfurl_cache_key(@@ -273,6 +292,7 @@ impl ShardService for UnfurlShard {nsfw,youtube_api_key.as_deref(),klipy_api_key.as_deref(),+ activity_pub_key_id.as_deref(),).await{@@ -324,6 +344,11 @@ impl ShardService for UnfurlShard {.unwrap_or_default();Ok(UnfurlResponse::Txt(records))}+ UnfurlRequest::ActivityPubKey => Ok(UnfurlResponse::ActivityPubKey(+ self.activity_pub_key+ .as_deref()+ .map(signature::public_key_base64),+ )),}}}diff --git a/fluxer_unfurl/src/types.rs b/fluxer_unfurl/src/types.rsindex 70c5ed823..387ded32d 100644--- a/fluxer_unfurl/src/types.rs+++ b/fluxer_unfurl/src/types.rs@@ -18,6 +18,8 @@ pub enum UnfurlRequest {youtube_api_key: Option<String>,#[serde(default)]klipy_api_key: Option<String>,+ #[serde(default)]+ activity_pub_key_id: Option<String>,},VerifyDomain {domain: String,@@ -34,6 +36,7 @@ pub enum UnfurlRequest {ResolveTxt {name: String,},+ ActivityPubKey,}#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]@@ -51,6 +54,7 @@ pub enum UnfurlResponse {Verified(bool),Fetched(FetchedResponse),Txt(Vec<String>),+ ActivityPubKey(Option<String>),Failed { message: String },}@@ -215,6 +219,7 @@ mod tests {cache_only,youtube_api_key,klipy_api_key,+ activity_pub_key_id,} => {assert_eq!(url, "https://fxtwitter.com/example/status/1");assert_eq!(nsfw_mode, Some(NsfwMode::Block));@@ -222,6 +227,7 @@ mod tests {assert!(!cache_only);assert!(youtube_api_key.is_none());assert!(klipy_api_key.is_none());+ assert!(activity_pub_key_id.is_none());}_ => panic!("expected an unfurl request"),}