ActivityPub-federated video streaming platform using P2P directly in your web browser
Support path style requests
Chocobozzzf4012a93838c
diff --git a/config/default.yaml b/config/default.yamlindex 56d604bac4..dce95440c8 100644--- a/config/default.yaml+++ b/config/default.yaml@@ -163,6 +163,9 @@ object_storage:region: 'us-east-1'+ # Use path style requests if the S3 provider does not support virtual hosting of buckets+ force_path_style: false+upload_acl:# Set this ACL on each uploaded object of public/unlisted videos# Use null if your S3 provider does not support object ACLdiff --git a/config/production.yaml.example b/config/production.yaml.exampleindex 00a09ad686..908f6f41d0 100644--- a/config/production.yaml.example+++ b/config/production.yaml.example@@ -161,6 +161,9 @@ object_storage:region: 'us-east-1'+ # Use path style requests if the S3 provider does not support virtual hosting of buckets+ force_path_style: false+upload_acl:# Set this ACL on each uploaded object of public/unlisted videos# Use null if your S3 provider does not support object ACLdiff --git a/packages/server-commands/src/server/config-command.ts b/packages/server-commands/src/server/config-command.tsindex 850daa336c..83c07a1f52 100644--- a/packages/server-commands/src/server/config-command.ts+++ b/packages/server-commands/src/server/config-command.ts@@ -231,12 +231,13 @@ export class ConfigCommand extends AbstractCommand {maxDuration?: numberalwaysTranscodeOriginalResolution?: boolean} = {}) {- const { allowReplay, transcoding, maxDuration, resolutions = 'min', alwaysTranscodeOriginalResolution } = options+ const { allowReplay, transcoding, maxDuration, resolutions = 'min', alwaysTranscodeOriginalResolution, maxUserLives } = optionsreturn this.updateExistingConfig({newConfig: {live: {enabled: true,+ maxUserLives,allowReplay,maxDuration,transcoding: {diff --git a/packages/server-commands/src/server/object-storage-command.ts b/packages/server-commands/src/server/object-storage-command.tsindex f5c1997692..7b02baea85 100644--- a/packages/server-commands/src/server/object-storage-command.ts+++ b/packages/server-commands/src/server/object-storage-command.ts@@ -72,26 +72,61 @@ export class ObjectStorageCommand {}}- getMockWebVideosBaseUrl () {- return `http://${this.getMockWebVideosBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`+ getMockWebVideosBaseUrl (options: {+ pathStyle?: boolean // default false+ } = {}) {+ const { pathStyle = false } = options++ return this.getMockFileBaseUrl({ bucketName: this.getMockWebVideosBucketName(), pathStyle })}- getMockPlaylistBaseUrl () {- return `http://${this.getMockStreamingPlaylistsBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`+ getMockPlaylistBaseUrl (options: {+ pathStyle?: boolean // default false+ } = {}) {+ const { pathStyle = false } = options++ return this.getMockFileBaseUrl({ bucketName: this.getMockStreamingPlaylistsBucketName(), pathStyle })}- getMockUserExportBaseUrl () {- return `http://${this.getMockUserExportBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`+ getMockUserExportBaseUrl (options: {+ pathStyle?: boolean // default false+ } = {}) {+ const { pathStyle = false } = options++ return this.getMockFileBaseUrl({ bucketName: this.getMockUserExportBucketName(), pathStyle })+ }++ getMockOriginalFileBaseUrl (options: {+ pathStyle?: boolean // default false+ } = {}) {+ const { pathStyle = false } = options++ return this.getMockFileBaseUrl({ bucketName: this.getMockOriginalFileBucketName(), pathStyle })}- getMockOriginalFileBaseUrl () {- return `http://${this.getMockOriginalFileBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`+ getMockCaptionFileBaseUrl (options: {+ pathStyle?: boolean // default false+ } = {}) {+ const { pathStyle = false } = options++ return this.getMockFileBaseUrl({ bucketName: this.getMockCaptionsBucketName(), pathStyle })}- getMockCaptionFileBaseUrl () {- return `http://${this.getMockCaptionsBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`+ private getMockFileBaseUrl (options: {+ bucketName: string+ pathStyle: boolean+ }) {+ const { bucketName, pathStyle } = options++ if (pathStyle) {+ return `http://${ObjectStorageCommand.getMockEndpointHost()}/${bucketName}/`+ }++ return `http://${bucketName}.${ObjectStorageCommand.getMockEndpointHost()}/`}+ // ---------------------------------------------------------------------------+async prepareDefaultMockBuckets () {await this.createMockBucket(this.getMockStreamingPlaylistsBucketName())await this.createMockBucket(this.getMockWebVideosBucketName())diff --git a/packages/tests/src/api/object-storage/live.ts b/packages/tests/src/api/object-storage/live.tsindex 8a89aadab1..ae77bb1582 100644--- a/packages/tests/src/api/object-storage/live.ts+++ b/packages/tests/src/api/object-storage/live.ts@@ -307,6 +307,65 @@ describe('Object storage for lives', function () {})})+ describe('With path style request', function () {+ let objectStorageBaseUrl: string++ before(async function () {+ this.timeout(120000)++ const bucketName = objectStorage.getMockStreamingPlaylistsBucketName()+ objectStorageBaseUrl = `http://${ObjectStorageCommand.getMockEndpointHost()}/${bucketName}`++ await objectStorage.prepareDefaultMockBuckets()++ const config = {+ object_storage: {+ enabled: true,+ endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),+ region: ObjectStorageCommand.getMockRegion(),++ credentials: ObjectStorageCommand.getMockCredentialsConfig(),+ force_path_style: true,++ streaming_playlists: {+ bucket_name: bucketName,+ prefix: '',+ store_live_streams: true+ }+ }+ }++ await servers[0].kill()+ await servers[0].run(config)++ await servers[0].config.enableLive({ transcoding: true, resolutions: 'min' })+ })++ it('Should publish a live with path style request', async function () {+ this.timeout(240000)++ const videoUUIDPermanent = await createLive(servers[0], true)++ const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDPermanent })+ await waitUntilLivePublishedOnAllServers(servers, videoUUIDPermanent)++ await testLiveVideoResolutions({+ originServer: servers[0],+ sqlCommand: sqlCommandServer1,+ servers,+ liveVideoId: videoUUIDPermanent,+ resolutions: [ 720 ],+ transcoded: true,+ objectStorage,+ objectStorageBaseUrl+ })++ await stopFfmpeg(ffmpegCommand)++ await servers[0].videos.remove({ id: videoUUIDPermanent })+ })+ })+describe('With live stream to object storage disabled', function () {let videoUUID: stringdiff --git a/packages/tests/src/api/object-storage/videos.ts b/packages/tests/src/api/object-storage/videos.tsindex 3c9e89e1aa..25d2a9eabe 100644--- a/packages/tests/src/api/object-storage/videos.ts+++ b/packages/tests/src/api/object-storage/videos.ts@@ -1,5 +1,6 @@/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */+import { getAllFiles } from '@peertube/peertube-core-utils'import { HttpStatusCode, VideoDetails } from '@peertube/peertube-models'import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'import {@@ -18,14 +19,32 @@ import { expectLogDoesNotContain, expectStartWith } from '@tests/shared/checks.jimport { checkTmpIsEmpty } from '@tests/shared/directories.js'import { generateHighBitrateVideo } from '@tests/shared/generate.js'import { MockObjectStorageProxy } from '@tests/shared/mock-servers/mock-object-storage.js'+import { checkWebTorrentWorks } from '@tests/shared/p2p.js'import { SQLCommand } from '@tests/shared/sql-command.js'import { checkPlaylistInfohash } from '@tests/shared/streaming-playlists.js'-import { checkWebTorrentWorks } from '@tests/shared/p2p.js'import bytes from 'bytes'import { expect } from 'chai'import { stat } from 'fs/promises'import merge from 'lodash-es/merge.js'+function buildBaseUrl (options: {+ baseMockUrl: string+ bucket: string+ pathStyle: boolean+}) {+ const { baseMockUrl, bucket, pathStyle } = options++ if (baseMockUrl) {+ if (pathStyle) return `${baseMockUrl}/${bucket}/${bucket}/`++ return `${baseMockUrl}/${bucket}/`+ }++ if (pathStyle) return `http://${ObjectStorageCommand.getMockEndpointHost()}/${bucket}/`++ return `http://${bucket}.${ObjectStorageCommand.getMockEndpointHost()}/`+}+async function checkFiles (options: {server: PeerTubeServeroriginServer: PeerTubeServer@@ -33,6 +52,8 @@ async function checkFiles (options: {video: VideoDetails+ pathStyle: boolean+baseMockUrl?: stringplaylistBucket: string@@ -50,15 +71,12 @@ async function checkFiles (options: {webVideoBucket,baseMockUrl,playlistPrefix,- webVideoPrefix+ webVideoPrefix,+ pathStyle} = options- let allFiles = video.files-for (const file of video.files) {- const baseUrl = baseMockUrl- ? `${baseMockUrl}/${webVideoBucket}/`- : `http://${webVideoBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`+ const baseUrl = buildBaseUrl({ baseMockUrl, bucket: webVideoBucket, pathStyle })const prefix = webVideoPrefix || ''const start = baseUrl + prefix@@ -75,11 +93,7 @@ async function checkFiles (options: {const hls = video.streamingPlaylists[0]if (hls) {- allFiles = allFiles.concat(hls.files)-- const baseUrl = baseMockUrl- ? `${baseMockUrl}/${playlistBucket}/`- : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`+ const baseUrl = buildBaseUrl({ baseMockUrl, bucket: playlistBucket, pathStyle })const prefix = playlistPrefix || ''const start = baseUrl + prefix@@ -107,6 +121,7 @@ async function checkFiles (options: {}}+ const allFiles = getAllFiles(video)for (const file of allFiles) {await checkWebTorrentWorks(file.magnetUri)@@ -118,6 +133,8 @@ async function checkFiles (options: {}function runTestSuite (options: {+ pathStyle: boolean+fixture?: stringmaxUploadPart?: string@@ -130,8 +147,9 @@ function runTestSuite (options: {useMockBaseUrl?: boolean}) {+ const { fixture, pathStyle, useMockBaseUrl } = options+const mockObjectStorageProxy = new MockObjectStorageProxy()- const { fixture } = optionslet baseMockUrl: stringlet servers: PeerTubeServer[]@@ -147,7 +165,7 @@ function runTestSuite (options: {this.timeout(240000)const port = await mockObjectStorageProxy.initialize()- baseMockUrl = options.useMockBaseUrl+ baseMockUrl = useMockBaseUrl? `http://127.0.0.1:${port}`: undefined@@ -160,6 +178,8 @@ function runTestSuite (options: {endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),region: ObjectStorageCommand.getMockRegion(),+ force_path_style: pathStyle,+credentials: ObjectStorageCommand.getMockCredentialsConfig(),max_upload_part: options.maxUploadPart || '5MB',@@ -364,7 +384,7 @@ describe('Object storage for videos', function () {await waitJobs([ server ], { skipDelayed: true })const video = await server.videos.get({ id: uuid })- expectStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl())+ expectStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl({ pathStyle: false }))})after(async function () {@@ -374,32 +394,75 @@ describe('Object storage for videos', function () {})})- describe('Test simple object storage', function () {- runTestSuite({- playlistBucket: objectStorage.getMockBucketName('streaming-playlists'),- webVideoBucket: objectStorage.getMockBucketName('web-videos')+ describe('Host style', function () {+ describe('Test simple object storage', function () {+ runTestSuite({+ playlistBucket: objectStorage.getMockBucketName('streaming-playlists'),+ webVideoBucket: objectStorage.getMockBucketName('web-videos'),+ pathStyle: false+ })})- })- describe('Test object storage with prefix', function () {- runTestSuite({- playlistBucket: objectStorage.getMockBucketName('mybucket'),- webVideoBucket: objectStorage.getMockBucketName('mybucket'),+ describe('Test object storage with prefix', function () {+ runTestSuite({+ playlistBucket: objectStorage.getMockBucketName('mybucket'),+ webVideoBucket: objectStorage.getMockBucketName('mybucket'),++ playlistPrefix: 'streaming-playlists_',+ webVideoPrefix: 'webvideo_',++ pathStyle: false+ })+ })- playlistPrefix: 'streaming-playlists_',- webVideoPrefix: 'webvideo_'+ describe('Test object storage with prefix and base URL', function () {+ runTestSuite({+ playlistBucket: objectStorage.getMockBucketName('mybucket'),+ webVideoBucket: objectStorage.getMockBucketName('mybucket'),++ playlistPrefix: 'streaming-playlists/',+ webVideoPrefix: 'webvideo/',++ useMockBaseUrl: true,++ pathStyle: false+ })})})- describe('Test object storage with prefix and base URL', function () {- runTestSuite({- playlistBucket: objectStorage.getMockBucketName('mybucket'),- webVideoBucket: objectStorage.getMockBucketName('mybucket'),+ describe('Path style', function () {+ describe('Test simple object storage', function () {+ runTestSuite({+ playlistBucket: objectStorage.getMockBucketName('streaming-playlists'),+ webVideoBucket: objectStorage.getMockBucketName('web-videos'),+ pathStyle: true+ })+ })++ describe('Test object storage with prefix', function () {+ runTestSuite({+ playlistBucket: objectStorage.getMockBucketName('mybucket'),+ webVideoBucket: objectStorage.getMockBucketName('mybucket'),++ playlistPrefix: 'streaming-playlists_',+ webVideoPrefix: 'webvideo_',++ pathStyle: true+ })+ })++ describe('Test object storage with prefix and base URL', function () {+ runTestSuite({+ playlistBucket: objectStorage.getMockBucketName('mybucket'),+ webVideoBucket: objectStorage.getMockBucketName('mybucket'),- playlistPrefix: 'streaming-playlists/',- webVideoPrefix: 'webvideo/',+ playlistPrefix: 'streaming-playlists/',+ webVideoPrefix: 'webvideo/',- useMockBaseUrl: true+ useMockBaseUrl: true,++ pathStyle: true+ })})})@@ -423,7 +486,8 @@ describe('Object storage for videos', function () {maxUploadPart,playlistBucket: objectStorage.getMockBucketName('streaming-playlists'),webVideoBucket: objectStorage.getMockBucketName('web-videos'),- fixture+ fixture,+ pathStyle: false})})})diff --git a/packages/tests/src/shared/mock-servers/mock-object-storage.ts b/packages/tests/src/shared/mock-servers/mock-object-storage.tsindex f97c57fd7c..c70cc5d306 100644--- a/packages/tests/src/shared/mock-servers/mock-object-storage.ts+++ b/packages/tests/src/shared/mock-servers/mock-object-storage.ts@@ -12,7 +12,13 @@ export class MockObjectStorageProxy {const app = express()app.get('/:bucketName/:path(*)', (req: express.Request, res: express.Response, next: express.NextFunction) => {- const url = `http://${req.params.bucketName}.${ObjectStorageCommand.getMockEndpointHost()}/${req.params.path}`+ const bucketName = req.params.bucketName++ let path = req.params.path+ // Some tests use `/:bucketName/:bucketName/:path` structure+ path = path.replace(new RegExp(`^${bucketName}/`), '')++ const url = `http://${bucketName}.${ObjectStorageCommand.getMockEndpointHost()}/${path}`if (process.env.DEBUG) {console.log('Receiving request on mocked server %s.', req.url)diff --git a/packages/transcription/tsconfig.json b/packages/transcription/tsconfig.jsonindex c390b2ae91..e6004d3e66 100644--- a/packages/transcription/tsconfig.json+++ b/packages/transcription/tsconfig.json@@ -8,7 +8,6 @@"references": [{ "path": "../models" },{ "path": "../core-utils" },- { "path": "../node-utils" },- { "path": "../server-commands" }+ { "path": "../node-utils" }]}diff --git a/server/core/initializers/checker-before-init.ts b/server/core/initializers/checker-before-init.tsindex c9022b0b36..7e2c32bbf9 100644--- a/server/core/initializers/checker-before-init.ts+++ b/server/core/initializers/checker-before-init.ts@@ -189,6 +189,7 @@ export function checkMissedConfig () {'object_storage.enabled','object_storage.endpoint','object_storage.region',+ 'object_storage.force_path_style','object_storage.upload_acl.public','object_storage.upload_acl.private','object_storage.proxy.proxify_private_files',diff --git a/server/core/initializers/config.ts b/server/core/initializers/config.tsindex 1797593cf7..21a788921f 100644--- a/server/core/initializers/config.ts+++ b/server/core/initializers/config.ts@@ -215,6 +215,7 @@ const CONFIG = {MAX_REQUEST_ATTEMPTS: config.get<number>('object_storage.max_request_attempts'),ENDPOINT: config.get<string>('object_storage.endpoint'),REGION: config.get<string>('object_storage.region'),+ FORCE_PATH_STYLE: config.get<boolean>('object_storage.force_path_style'),UPLOAD_ACL: {PUBLIC: config.get<string>('object_storage.upload_acl.public'),PRIVATE: config.get<string>('object_storage.upload_acl.private')diff --git a/server/core/lib/object-storage/shared/client.ts b/server/core/lib/object-storage/shared/client.tsindex 0dacb5667b..9f3605af00 100644--- a/server/core/lib/object-storage/shared/client.ts+++ b/server/core/lib/object-storage/shared/client.ts@@ -47,6 +47,8 @@ function getClient () {: undefined,requestHandler: await getProxyRequestHandler(),maxAttempts: CONFIG.OBJECT_STORAGE.MAX_REQUEST_ATTEMPTS,+ forcePathStyle: CONFIG.OBJECT_STORAGE.FORCE_PATH_STYLE,+// Default behaviour has incompatibilities with some S3 providers: https://github.com/aws/aws-sdk-js-v3/issues/6810requestChecksumCalculation: 'WHEN_REQUIRED',responseChecksumValidation: 'WHEN_REQUIRED'diff --git a/server/core/lib/object-storage/urls.ts b/server/core/lib/object-storage/urls.tsindex ca185710c0..a6df19213f 100644--- a/server/core/lib/object-storage/urls.ts+++ b/server/core/lib/object-storage/urls.ts@@ -1,3 +1,4 @@+import { CONFIG } from '@server/initializers/config.js'import { OBJECT_STORAGE_PROXY_PATHS, WEBSERVER } from '@server/initializers/constants.js'import { MVideoUUID } from '@server/types/models/index.js'import { BucketInfo, buildKey, getEndpointParsed } from './shared/index.js'@@ -24,11 +25,18 @@ export function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) {export function getWebVideoPrivateFileUrl (filename: string) {return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEB_VIDEOS + filename}++// ---------------------------------------------------------------------------+// Private// ---------------------------------------------------------------------------function getBaseUrl (bucketInfo: BucketInfo, baseUrl?: string) {if (baseUrl) return baseUrl+ if (CONFIG.OBJECT_STORAGE.FORCE_PATH_STYLE) {+ return `${getEndpointParsed().protocol}//${getEndpointParsed().host}/${bucketInfo.BUCKET_NAME}/`+ }+return `${getEndpointParsed().protocol}//${bucketInfo.BUCKET_NAME}.${getEndpointParsed().host}/`}diff --git a/support/docker/production/.env b/support/docker/production/.envindex 1f1f2f1031..3c53acc968 100644--- a/support/docker/production/.env+++ b/support/docker/production/.env@@ -51,6 +51,7 @@ OPENDKIM_RequireSafeKeys=no#PEERTUBE_OBJECT_STORAGE_ENABLED=#PEERTUBE_OBJECT_STORAGE_ENDPOINT=#PEERTUBE_OBJECT_STORAGE_REGION=+#PEERTUBE_OBJECT_STORAGE_FORCE_PATH_STYLE=#PEERTUBE_OBJECT_STORAGE_CREDENTIALS_ACCESS_KEY_ID=#PEERTUBE_OBJECT_STORAGE_CREDENTIALS_SECRET_ACCESS_KEY=#PEERTUBE_OBJECT_STORAGE_STREAMING_PLAYLISTS_BUCKET_NAME=diff --git a/support/docker/production/config/custom-environment-variables.yaml b/support/docker/production/config/custom-environment-variables.yamlindex af0fd2053c..d47026729a 100644--- a/support/docker/production/config/custom-environment-variables.yaml+++ b/support/docker/production/config/custom-environment-variables.yaml@@ -68,6 +68,10 @@ object_storage:region: "PEERTUBE_OBJECT_STORAGE_REGION"+ force_path_style:+ __name: "PEERTUBE_OBJECT_STORAGE_FORCE_PATH_STYLE"+ __format: "json"+upload_acl:public: "PEERTUBE_OBJECT_STORAGE_UPLOAD_ACL_PUBLIC"private: "PEERTUBE_OBJECT_STORAGE_UPLOAD_ACL_PRIVATE"