Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Support path style requests

Chocobozzzf4012a93838c
diff --git a/config/default.yaml b/config/default.yaml
index 56d604bac4..dce95440c8 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -163,6 +163,9 @@ object_storage:
region: 'us-east-1'
+ # Use path style requests if the S3 provider does not support virtual hosting of buckets
+ force_path_style: false
+
upload_acl:
# Set this ACL on each uploaded object of public/unlisted videos
# Use null if your S3 provider does not support object ACL
diff --git a/config/production.yaml.example b/config/production.yaml.example
index 00a09ad686..908f6f41d0 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -161,6 +161,9 @@ object_storage:
region: 'us-east-1'
+ # Use path style requests if the S3 provider does not support virtual hosting of buckets
+ force_path_style: false
+
upload_acl:
# Set this ACL on each uploaded object of public/unlisted videos
# Use null if your S3 provider does not support object ACL
diff --git a/packages/server-commands/src/server/config-command.ts b/packages/server-commands/src/server/config-command.ts
index 850daa336c..83c07a1f52 100644
--- a/packages/server-commands/src/server/config-command.ts
+++ b/packages/server-commands/src/server/config-command.ts
@@ -231,12 +231,13 @@ export class ConfigCommand extends AbstractCommand {
maxDuration?: number
alwaysTranscodeOriginalResolution?: boolean
} = {}) {
- const { allowReplay, transcoding, maxDuration, resolutions = 'min', alwaysTranscodeOriginalResolution } = options
+ const { allowReplay, transcoding, maxDuration, resolutions = 'min', alwaysTranscodeOriginalResolution, maxUserLives } = options
return this.updateExistingConfig({
newConfig: {
live: {
enabled: true,
+ maxUserLives,
allowReplay,
maxDuration,
transcoding: {
diff --git a/packages/server-commands/src/server/object-storage-command.ts b/packages/server-commands/src/server/object-storage-command.ts
index f5c1997692..7b02baea85 100644
--- a/packages/server-commands/src/server/object-storage-command.ts
+++ b/packages/server-commands/src/server/object-storage-command.ts
@@ -72,26 +72,61 @@ export class ObjectStorageCommand {
}
}
- getMockWebVideosBaseUrl () {
- return `http://${this.getMockWebVideosBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`
+ getMockWebVideosBaseUrl (options: {
+ pathStyle?: boolean // default false
+ } = {}) {
+ const { pathStyle = false } = options
+
+ return this.getMockFileBaseUrl({ bucketName: this.getMockWebVideosBucketName(), pathStyle })
}
- getMockPlaylistBaseUrl () {
- return `http://${this.getMockStreamingPlaylistsBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`
+ getMockPlaylistBaseUrl (options: {
+ pathStyle?: boolean // default false
+ } = {}) {
+ const { pathStyle = false } = options
+
+ return this.getMockFileBaseUrl({ bucketName: this.getMockStreamingPlaylistsBucketName(), pathStyle })
}
- getMockUserExportBaseUrl () {
- return `http://${this.getMockUserExportBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`
+ getMockUserExportBaseUrl (options: {
+ pathStyle?: boolean // default false
+ } = {}) {
+ const { pathStyle = false } = options
+
+ return this.getMockFileBaseUrl({ bucketName: this.getMockUserExportBucketName(), pathStyle })
+ }
+
+ getMockOriginalFileBaseUrl (options: {
+ pathStyle?: boolean // default false
+ } = {}) {
+ const { pathStyle = false } = options
+
+ return this.getMockFileBaseUrl({ bucketName: this.getMockOriginalFileBucketName(), pathStyle })
}
- getMockOriginalFileBaseUrl () {
- return `http://${this.getMockOriginalFileBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`
+ getMockCaptionFileBaseUrl (options: {
+ pathStyle?: boolean // default false
+ } = {}) {
+ const { pathStyle = false } = options
+
+ return this.getMockFileBaseUrl({ bucketName: this.getMockCaptionsBucketName(), pathStyle })
}
- getMockCaptionFileBaseUrl () {
- return `http://${this.getMockCaptionsBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`
+ private getMockFileBaseUrl (options: {
+ bucketName: string
+ pathStyle: boolean
+ }) {
+ const { bucketName, pathStyle } = options
+
+ if (pathStyle) {
+ return `http://${ObjectStorageCommand.getMockEndpointHost()}/${bucketName}/`
+ }
+
+ return `http://${bucketName}.${ObjectStorageCommand.getMockEndpointHost()}/`
}
+ // ---------------------------------------------------------------------------
+
async prepareDefaultMockBuckets () {
await this.createMockBucket(this.getMockStreamingPlaylistsBucketName())
await this.createMockBucket(this.getMockWebVideosBucketName())
diff --git a/packages/tests/src/api/object-storage/live.ts b/packages/tests/src/api/object-storage/live.ts
index 8a89aadab1..ae77bb1582 100644
--- a/packages/tests/src/api/object-storage/live.ts
+++ b/packages/tests/src/api/object-storage/live.ts
@@ -307,6 +307,65 @@ describe('Object storage for lives', function () {
})
})
+ describe('With path style request', function () {
+ let objectStorageBaseUrl: string
+
+ before(async function () {
+ this.timeout(120000)
+
+ const bucketName = objectStorage.getMockStreamingPlaylistsBucketName()
+ objectStorageBaseUrl = `http://${ObjectStorageCommand.getMockEndpointHost()}/${bucketName}`
+
+ await objectStorage.prepareDefaultMockBuckets()
+
+ const config = {
+ object_storage: {
+ enabled: true,
+ endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
+ region: ObjectStorageCommand.getMockRegion(),
+
+ credentials: ObjectStorageCommand.getMockCredentialsConfig(),
+ force_path_style: true,
+
+ streaming_playlists: {
+ bucket_name: bucketName,
+ prefix: '',
+ store_live_streams: true
+ }
+ }
+ }
+
+ await servers[0].kill()
+ await servers[0].run(config)
+
+ await servers[0].config.enableLive({ transcoding: true, resolutions: 'min' })
+ })
+
+ it('Should publish a live with path style request', async function () {
+ this.timeout(240000)
+
+ const videoUUIDPermanent = await createLive(servers[0], true)
+
+ const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDPermanent })
+ await waitUntilLivePublishedOnAllServers(servers, videoUUIDPermanent)
+
+ await testLiveVideoResolutions({
+ originServer: servers[0],
+ sqlCommand: sqlCommandServer1,
+ servers,
+ liveVideoId: videoUUIDPermanent,
+ resolutions: [ 720 ],
+ transcoded: true,
+ objectStorage,
+ objectStorageBaseUrl
+ })
+
+ await stopFfmpeg(ffmpegCommand)
+
+ await servers[0].videos.remove({ id: videoUUIDPermanent })
+ })
+ })
+
describe('With live stream to object storage disabled', function () {
let videoUUID: string
diff --git a/packages/tests/src/api/object-storage/videos.ts b/packages/tests/src/api/object-storage/videos.ts
index 3c9e89e1aa..25d2a9eabe 100644
--- a/packages/tests/src/api/object-storage/videos.ts
+++ b/packages/tests/src/api/object-storage/videos.ts
@@ -1,5 +1,6 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+import { getAllFiles } from '@peertube/peertube-core-utils'
import { HttpStatusCode, VideoDetails } from '@peertube/peertube-models'
import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
import {
@@ -18,14 +19,32 @@ import { expectLogDoesNotContain, expectStartWith } from '@tests/shared/checks.j
import { checkTmpIsEmpty } from '@tests/shared/directories.js'
import { generateHighBitrateVideo } from '@tests/shared/generate.js'
import { MockObjectStorageProxy } from '@tests/shared/mock-servers/mock-object-storage.js'
+import { checkWebTorrentWorks } from '@tests/shared/p2p.js'
import { SQLCommand } from '@tests/shared/sql-command.js'
import { checkPlaylistInfohash } from '@tests/shared/streaming-playlists.js'
-import { checkWebTorrentWorks } from '@tests/shared/p2p.js'
import bytes from 'bytes'
import { expect } from 'chai'
import { stat } from 'fs/promises'
import merge from 'lodash-es/merge.js'
+function buildBaseUrl (options: {
+ baseMockUrl: string
+ bucket: string
+ pathStyle: boolean
+}) {
+ const { baseMockUrl, bucket, pathStyle } = options
+
+ if (baseMockUrl) {
+ if (pathStyle) return `${baseMockUrl}/${bucket}/${bucket}/`
+
+ return `${baseMockUrl}/${bucket}/`
+ }
+
+ if (pathStyle) return `http://${ObjectStorageCommand.getMockEndpointHost()}/${bucket}/`
+
+ return `http://${bucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
+}
+
async function checkFiles (options: {
server: PeerTubeServer
originServer: PeerTubeServer
@@ -33,6 +52,8 @@ async function checkFiles (options: {
video: VideoDetails
+ pathStyle: boolean
+
baseMockUrl?: string
playlistBucket: string
@@ -50,15 +71,12 @@ async function checkFiles (options: {
webVideoBucket,
baseMockUrl,
playlistPrefix,
- webVideoPrefix
+ webVideoPrefix,
+ pathStyle
} = options
- let allFiles = video.files
-
for (const file of video.files) {
- const baseUrl = baseMockUrl
- ? `${baseMockUrl}/${webVideoBucket}/`
- : `http://${webVideoBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
+ const baseUrl = buildBaseUrl({ baseMockUrl, bucket: webVideoBucket, pathStyle })
const prefix = webVideoPrefix || ''
const start = baseUrl + prefix
@@ -75,11 +93,7 @@ async function checkFiles (options: {
const hls = video.streamingPlaylists[0]
if (hls) {
- allFiles = allFiles.concat(hls.files)
-
- const baseUrl = baseMockUrl
- ? `${baseMockUrl}/${playlistBucket}/`
- : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
+ const baseUrl = buildBaseUrl({ baseMockUrl, bucket: playlistBucket, pathStyle })
const prefix = playlistPrefix || ''
const start = baseUrl + prefix
@@ -107,6 +121,7 @@ async function checkFiles (options: {
}
}
+ const allFiles = getAllFiles(video)
for (const file of allFiles) {
await checkWebTorrentWorks(file.magnetUri)
@@ -118,6 +133,8 @@ async function checkFiles (options: {
}
function runTestSuite (options: {
+ pathStyle: boolean
+
fixture?: string
maxUploadPart?: string
@@ -130,8 +147,9 @@ function runTestSuite (options: {
useMockBaseUrl?: boolean
}) {
+ const { fixture, pathStyle, useMockBaseUrl } = options
+
const mockObjectStorageProxy = new MockObjectStorageProxy()
- const { fixture } = options
let baseMockUrl: string
let servers: PeerTubeServer[]
@@ -147,7 +165,7 @@ function runTestSuite (options: {
this.timeout(240000)
const port = await mockObjectStorageProxy.initialize()
- baseMockUrl = options.useMockBaseUrl
+ baseMockUrl = useMockBaseUrl
? `http://127.0.0.1:${port}`
: undefined
@@ -160,6 +178,8 @@ function runTestSuite (options: {
endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
region: ObjectStorageCommand.getMockRegion(),
+ force_path_style: pathStyle,
+
credentials: ObjectStorageCommand.getMockCredentialsConfig(),
max_upload_part: options.maxUploadPart || '5MB',
@@ -364,7 +384,7 @@ describe('Object storage for videos', function () {
await waitJobs([ server ], { skipDelayed: true })
const video = await server.videos.get({ id: uuid })
- expectStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl())
+ expectStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl({ pathStyle: false }))
})
after(async function () {
@@ -374,32 +394,75 @@ describe('Object storage for videos', function () {
})
})
- describe('Test simple object storage', function () {
- runTestSuite({
- playlistBucket: objectStorage.getMockBucketName('streaming-playlists'),
- webVideoBucket: objectStorage.getMockBucketName('web-videos')
+ describe('Host style', function () {
+ describe('Test simple object storage', function () {
+ runTestSuite({
+ playlistBucket: objectStorage.getMockBucketName('streaming-playlists'),
+ webVideoBucket: objectStorage.getMockBucketName('web-videos'),
+ pathStyle: false
+ })
})
- })
- describe('Test object storage with prefix', function () {
- runTestSuite({
- playlistBucket: objectStorage.getMockBucketName('mybucket'),
- webVideoBucket: objectStorage.getMockBucketName('mybucket'),
+ describe('Test object storage with prefix', function () {
+ runTestSuite({
+ playlistBucket: objectStorage.getMockBucketName('mybucket'),
+ webVideoBucket: objectStorage.getMockBucketName('mybucket'),
+
+ playlistPrefix: 'streaming-playlists_',
+ webVideoPrefix: 'webvideo_',
+
+ pathStyle: false
+ })
+ })
- playlistPrefix: 'streaming-playlists_',
- webVideoPrefix: 'webvideo_'
+ describe('Test object storage with prefix and base URL', function () {
+ runTestSuite({
+ playlistBucket: objectStorage.getMockBucketName('mybucket'),
+ webVideoBucket: objectStorage.getMockBucketName('mybucket'),
+
+ playlistPrefix: 'streaming-playlists/',
+ webVideoPrefix: 'webvideo/',
+
+ useMockBaseUrl: true,
+
+ pathStyle: false
+ })
})
})
- describe('Test object storage with prefix and base URL', function () {
- runTestSuite({
- playlistBucket: objectStorage.getMockBucketName('mybucket'),
- webVideoBucket: objectStorage.getMockBucketName('mybucket'),
+ describe('Path style', function () {
+ describe('Test simple object storage', function () {
+ runTestSuite({
+ playlistBucket: objectStorage.getMockBucketName('streaming-playlists'),
+ webVideoBucket: objectStorage.getMockBucketName('web-videos'),
+ pathStyle: true
+ })
+ })
+
+ describe('Test object storage with prefix', function () {
+ runTestSuite({
+ playlistBucket: objectStorage.getMockBucketName('mybucket'),
+ webVideoBucket: objectStorage.getMockBucketName('mybucket'),
+
+ playlistPrefix: 'streaming-playlists_',
+ webVideoPrefix: 'webvideo_',
+
+ pathStyle: true
+ })
+ })
+
+ describe('Test object storage with prefix and base URL', function () {
+ runTestSuite({
+ playlistBucket: objectStorage.getMockBucketName('mybucket'),
+ webVideoBucket: objectStorage.getMockBucketName('mybucket'),
- playlistPrefix: 'streaming-playlists/',
- webVideoPrefix: 'webvideo/',
+ playlistPrefix: 'streaming-playlists/',
+ webVideoPrefix: 'webvideo/',
- useMockBaseUrl: true
+ useMockBaseUrl: true,
+
+ pathStyle: true
+ })
})
})
@@ -423,7 +486,8 @@ describe('Object storage for videos', function () {
maxUploadPart,
playlistBucket: objectStorage.getMockBucketName('streaming-playlists'),
webVideoBucket: objectStorage.getMockBucketName('web-videos'),
- fixture
+ fixture,
+ pathStyle: false
})
})
})
diff --git a/packages/tests/src/shared/mock-servers/mock-object-storage.ts b/packages/tests/src/shared/mock-servers/mock-object-storage.ts
index f97c57fd7c..c70cc5d306 100644
--- a/packages/tests/src/shared/mock-servers/mock-object-storage.ts
+++ b/packages/tests/src/shared/mock-servers/mock-object-storage.ts
@@ -12,7 +12,13 @@ export class MockObjectStorageProxy {
const app = express()
app.get('/:bucketName/:path(*)', (req: express.Request, res: express.Response, next: express.NextFunction) => {
- const url = `http://${req.params.bucketName}.${ObjectStorageCommand.getMockEndpointHost()}/${req.params.path}`
+ const bucketName = req.params.bucketName
+
+ let path = req.params.path
+ // Some tests use `/:bucketName/:bucketName/:path` structure
+ path = path.replace(new RegExp(`^${bucketName}/`), '')
+
+ const url = `http://${bucketName}.${ObjectStorageCommand.getMockEndpointHost()}/${path}`
if (process.env.DEBUG) {
console.log('Receiving request on mocked server %s.', req.url)
diff --git a/packages/transcription/tsconfig.json b/packages/transcription/tsconfig.json
index c390b2ae91..e6004d3e66 100644
--- a/packages/transcription/tsconfig.json
+++ b/packages/transcription/tsconfig.json
@@ -8,7 +8,6 @@
"references": [
{ "path": "../models" },
{ "path": "../core-utils" },
- { "path": "../node-utils" },
- { "path": "../server-commands" }
+ { "path": "../node-utils" }
]
}
diff --git a/server/core/initializers/checker-before-init.ts b/server/core/initializers/checker-before-init.ts
index c9022b0b36..7e2c32bbf9 100644
--- a/server/core/initializers/checker-before-init.ts
+++ b/server/core/initializers/checker-before-init.ts
@@ -189,6 +189,7 @@ export function checkMissedConfig () {
'object_storage.enabled',
'object_storage.endpoint',
'object_storage.region',
+ 'object_storage.force_path_style',
'object_storage.upload_acl.public',
'object_storage.upload_acl.private',
'object_storage.proxy.proxify_private_files',
diff --git a/server/core/initializers/config.ts b/server/core/initializers/config.ts
index 1797593cf7..21a788921f 100644
--- a/server/core/initializers/config.ts
+++ b/server/core/initializers/config.ts
@@ -215,6 +215,7 @@ const CONFIG = {
MAX_REQUEST_ATTEMPTS: config.get<number>('object_storage.max_request_attempts'),
ENDPOINT: config.get<string>('object_storage.endpoint'),
REGION: config.get<string>('object_storage.region'),
+ FORCE_PATH_STYLE: config.get<boolean>('object_storage.force_path_style'),
UPLOAD_ACL: {
PUBLIC: config.get<string>('object_storage.upload_acl.public'),
PRIVATE: config.get<string>('object_storage.upload_acl.private')
diff --git a/server/core/lib/object-storage/shared/client.ts b/server/core/lib/object-storage/shared/client.ts
index 0dacb5667b..9f3605af00 100644
--- a/server/core/lib/object-storage/shared/client.ts
+++ b/server/core/lib/object-storage/shared/client.ts
@@ -47,6 +47,8 @@ function getClient () {
: undefined,
requestHandler: await getProxyRequestHandler(),
maxAttempts: CONFIG.OBJECT_STORAGE.MAX_REQUEST_ATTEMPTS,
+ forcePathStyle: CONFIG.OBJECT_STORAGE.FORCE_PATH_STYLE,
+
// Default behaviour has incompatibilities with some S3 providers: https://github.com/aws/aws-sdk-js-v3/issues/6810
requestChecksumCalculation: 'WHEN_REQUIRED',
responseChecksumValidation: 'WHEN_REQUIRED'
diff --git a/server/core/lib/object-storage/urls.ts b/server/core/lib/object-storage/urls.ts
index ca185710c0..a6df19213f 100644
--- a/server/core/lib/object-storage/urls.ts
+++ b/server/core/lib/object-storage/urls.ts
@@ -1,3 +1,4 @@
+import { CONFIG } from '@server/initializers/config.js'
import { OBJECT_STORAGE_PROXY_PATHS, WEBSERVER } from '@server/initializers/constants.js'
import { MVideoUUID } from '@server/types/models/index.js'
import { BucketInfo, buildKey, getEndpointParsed } from './shared/index.js'
@@ -24,11 +25,18 @@ export function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) {
export function getWebVideoPrivateFileUrl (filename: string) {
return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEB_VIDEOS + filename
}
+
+// ---------------------------------------------------------------------------
+// Private
// ---------------------------------------------------------------------------
function getBaseUrl (bucketInfo: BucketInfo, baseUrl?: string) {
if (baseUrl) return baseUrl
+ if (CONFIG.OBJECT_STORAGE.FORCE_PATH_STYLE) {
+ return `${getEndpointParsed().protocol}//${getEndpointParsed().host}/${bucketInfo.BUCKET_NAME}/`
+ }
+
return `${getEndpointParsed().protocol}//${bucketInfo.BUCKET_NAME}.${getEndpointParsed().host}/`
}
diff --git a/support/docker/production/.env b/support/docker/production/.env
index 1f1f2f1031..3c53acc968 100644
--- a/support/docker/production/.env
+++ b/support/docker/production/.env
@@ -51,6 +51,7 @@ OPENDKIM_RequireSafeKeys=no
#PEERTUBE_OBJECT_STORAGE_ENABLED=
#PEERTUBE_OBJECT_STORAGE_ENDPOINT=
#PEERTUBE_OBJECT_STORAGE_REGION=
+#PEERTUBE_OBJECT_STORAGE_FORCE_PATH_STYLE=
#PEERTUBE_OBJECT_STORAGE_CREDENTIALS_ACCESS_KEY_ID=
#PEERTUBE_OBJECT_STORAGE_CREDENTIALS_SECRET_ACCESS_KEY=
#PEERTUBE_OBJECT_STORAGE_STREAMING_PLAYLISTS_BUCKET_NAME=
diff --git a/support/docker/production/config/custom-environment-variables.yaml b/support/docker/production/config/custom-environment-variables.yaml
index af0fd2053c..d47026729a 100644
--- a/support/docker/production/config/custom-environment-variables.yaml
+++ b/support/docker/production/config/custom-environment-variables.yaml
@@ -68,6 +68,10 @@ object_storage:
region: "PEERTUBE_OBJECT_STORAGE_REGION"
+ force_path_style:
+ __name: "PEERTUBE_OBJECT_STORAGE_FORCE_PATH_STYLE"
+ __format: "json"
+
upload_acl:
public: "PEERTUBE_OBJECT_STORAGE_UPLOAD_ACL_PUBLIC"
private: "PEERTUBE_OBJECT_STORAGE_UPLOAD_ACL_PRIVATE"