Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Add cookies to yt-dlp config and docker env (#7510)

Viny Brun Kriesangfb46151616a1
diff --git a/config/default.yaml b/config/default.yaml
index b33871bdf8..db2c208b4f 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -884,6 +884,15 @@ import:
proxies:
# - "https://username:[email protected]:8888"
+ cookies:
+ # Enable this to pass cookies to yt-dlp. This can help when imports need a real browser session or when your server IP is limited
+ # Only enable this if you trust users allowed to trigger imports because PeerTube will import using the account from which these cookies were exported
+ # That account may be rate limited or temporarily blocked
+ # PeerTube expects a Netscape-format file at storage/tmp-persistent/youtube-cookies.txt
+ # See https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for cookie export instructions
+ # See https://docs.joinpeertube.org/maintain/configuration#cookies-for-youtube-imports for PeerTube-specific setup
+ enabled: false
+
# Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)
torrent:
# We recommend to only enable magnet URI/torrent import if you trust your users
diff --git a/config/production.yaml.example b/config/production.yaml.example
index c37b64e2da..92655d9146 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -894,6 +894,15 @@ import:
proxies:
# - "https://username:[email protected]:8888"
+ cookies:
+ # Enable this to pass cookies to yt-dlp. This can help when imports need a real browser session or when your server IP is limited
+ # Only enable this if you trust users allowed to trigger imports because PeerTube will import using the account from which these cookies were exported
+ # That account may be rate limited or temporarily blocked
+ # PeerTube expects a Netscape-format file at /var/www/peertube/storage/tmp-persistent/youtube-cookies.txt
+ # See https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for cookie export instructions
+ # See https://docs.joinpeertube.org/maintain/configuration#cookies-for-youtube-imports for PeerTube-specific setup
+ enabled: false
+
# Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)
torrent:
# We recommend to only enable magnet URI/torrent import if you trust your users
diff --git a/packages/tests/src/server-helpers/youtube-dl.ts b/packages/tests/src/server-helpers/youtube-dl.ts
index f4922d5e70..25451f8364 100644
--- a/packages/tests/src/server-helpers/youtube-dl.ts
+++ b/packages/tests/src/server-helpers/youtube-dl.ts
@@ -1,7 +1,11 @@
/* oxlint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { expect } from 'chai'
+import { mkdtemp, rm, writeFile } from 'fs/promises'
+import { tmpdir } from 'os'
+import { join } from 'path'
import { YoutubeDLCLI } from '@peertube/peertube-server/core/helpers/youtube-dl/youtube-dl-cli.js'
+import { logger } from '@peertube/peertube-server/core/helpers/logger.js'
import { CONFIG } from '@peertube/peertube-server/core/initializers/config.js'
describe('YoutubeDLCLI', function () {
@@ -91,4 +95,96 @@ describe('YoutubeDLCLI', function () {
}
})
})
+
+ describe('wrapWithCookiesOptions', function () {
+ let cli: any
+
+ before(function () {
+ cli = Object.create(YoutubeDLCLI.prototype)
+ })
+
+ it('Should prepend cookies file when configured and the file exists', async function () {
+ const originalTmpPersistentDirDescriptor = Object.getOwnPropertyDescriptor(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR')
+ const originalCookiesEnabledDescriptor = Object.getOwnPropertyDescriptor(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED')
+ const tempDir = await mkdtemp(join(tmpdir(), 'peertube-cookies-'))
+ const cookiesFile = join(tempDir, 'youtube-cookies.txt')
+
+ await writeFile(cookiesFile, '# Netscape HTTP Cookie File\n')
+
+ Object.defineProperty(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR', {
+ get: () => tempDir,
+ configurable: true
+ })
+
+ Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', {
+ get: () => true,
+ configurable: true
+ })
+
+ try {
+ const inputArgs = [ '--dump-json', '-f', 'best' ]
+ const result: string[] = await cli.wrapWithCookiesOptions(inputArgs)
+
+ expect(result).to.deep.equal([ '--cookies', cookiesFile, ...inputArgs ])
+ } finally {
+ Object.defineProperty(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR', originalTmpPersistentDirDescriptor)
+ Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', originalCookiesEnabledDescriptor)
+ await rm(tempDir, { recursive: true, force: true })
+ }
+ })
+
+ it('Should log an error and continue when the cookies file is missing', async function () {
+ const originalTmpPersistentDirDescriptor = Object.getOwnPropertyDescriptor(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR')
+ const originalCookiesEnabledDescriptor = Object.getOwnPropertyDescriptor(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED')
+ const originalLoggerError = logger.error
+ const tempDir = await mkdtemp(join(tmpdir(), 'peertube-cookies-'))
+ const loggedMessages: any[][] = []
+
+ Object.defineProperty(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR', {
+ get: () => tempDir,
+ configurable: true
+ })
+
+ Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', {
+ get: () => true,
+ configurable: true
+ })
+
+ ;(logger as any).error = (...args: any[]) => {
+ loggedMessages.push(args)
+ }
+
+ try {
+ const inputArgs = [ '--dump-json', '-f', 'best' ]
+ const result: string[] = await cli.wrapWithCookiesOptions(inputArgs)
+
+ expect(result).to.deep.equal(inputArgs)
+ expect(loggedMessages).to.have.lengthOf(1)
+ expect(loggedMessages[0][0]).to.contain('yt-dlp cookies are enabled but the cookies file %s does not exist')
+ } finally {
+ Object.defineProperty(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR', originalTmpPersistentDirDescriptor)
+ Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', originalCookiesEnabledDescriptor)
+ ;(logger as any).error = originalLoggerError
+ await rm(tempDir, { recursive: true, force: true })
+ }
+ })
+
+ it('Should not modify args when cookies are disabled', async function () {
+ const originalCookiesEnabledDescriptor = Object.getOwnPropertyDescriptor(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED')
+
+ Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', {
+ get: () => false,
+ configurable: true
+ })
+
+ try {
+ const inputArgs = [ '--dump-json', '-f', 'best' ]
+ const result: string[] = await cli.wrapWithCookiesOptions(inputArgs)
+
+ expect(result).to.deep.equal(inputArgs)
+ } finally {
+ Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', originalCookiesEnabledDescriptor)
+ }
+ })
+ })
})
diff --git a/server/core/helpers/youtube-dl/youtube-dl-cli.ts b/server/core/helpers/youtube-dl/youtube-dl-cli.ts
index 299a3be988..3a6e2f7251 100644
--- a/server/core/helpers/youtube-dl/youtube-dl-cli.ts
+++ b/server/core/helpers/youtube-dl/youtube-dl-cli.ts
@@ -215,6 +215,7 @@ export class YoutubeDLCLI {
let completeArgs = this.wrapWithJSRuntimeOptions(args)
completeArgs = this.wrapWithProxyOptions(completeArgs)
+ completeArgs = await this.wrapWithCookiesOptions(completeArgs)
completeArgs = this.wrapWithIPOptions(completeArgs)
completeArgs = this.wrapWithFFmpegOptions(completeArgs)
@@ -272,6 +273,28 @@ export class YoutubeDLCLI {
return args
}
+ private async wrapWithCookiesOptions (args: string[]) {
+ if (!CONFIG.IMPORT.VIDEOS.HTTP.COOKIES.ENABLED) {
+ return args
+ }
+
+ const cookiesPath = join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, 'youtube-cookies.txt')
+
+ if (!await pathExists(cookiesPath)) {
+ logger.error(
+ 'yt-dlp cookies are enabled but the cookies file %s does not exist. Continuing without cookies.',
+ cookiesPath,
+ lTags()
+ )
+
+ return args
+ }
+
+ logger.debug('Using cookies file %s for YoutubeDL', cookiesPath, lTags())
+
+ return [ '--cookies', cookiesPath ].concat(args)
+ }
+
private wrapWithFFmpegOptions (args: string[]) {
if (process.env.FFMPEG_PATH) {
logger.debug('Using ffmpeg location %s for YoutubeDL', process.env.FFMPEG_PATH, lTags())
diff --git a/server/core/initializers/checker-before-init.ts b/server/core/initializers/checker-before-init.ts
index 54e338e106..b8c45fe279 100644
--- a/server/core/initializers/checker-before-init.ts
+++ b/server/core/initializers/checker-before-init.ts
@@ -119,6 +119,7 @@ export function checkMissedConfig () {
'import.videos.timeout',
'import.videos.http.force_ipv4',
'import.videos.http.proxies',
+ 'import.videos.http.cookies.enabled',
'import.video_channel_synchronization.enabled',
'import.video_channel_synchronization.max_per_user',
'import.video_channel_synchronization.check_interval',
diff --git a/server/core/initializers/config.ts b/server/core/initializers/config.ts
index 49fecd9c1c..e32b5524f7 100644
--- a/server/core/initializers/config.ts
+++ b/server/core/initializers/config.ts
@@ -909,6 +909,12 @@ const CONFIG = {
get PROXIES () {
return config.get<string[]>('import.videos.http.proxies')
+ },
+
+ COOKIES: {
+ get ENABLED () {
+ return config.get<boolean>('import.videos.http.cookies.enabled')
+ }
}
},
TORRENT: {
diff --git a/support/docker/production/.env b/support/docker/production/.env
index c1b872ee48..208280ead8 100644
--- a/support/docker/production/.env
+++ b/support/docker/production/.env
@@ -102,6 +102,34 @@ PEERTUBE_OBJECT_STORAGE_UPLOAD_ACL_PUBLIC="public-read"
PEERTUBE_OBJECT_STORAGE_UPLOAD_ACL_PRIVATE="private"
+# -----------------------------------------------------------------------------
+# Outbound Proxies (Optional)
+# -----------------------------------------------------------------------------
+# These proxies affect all outbound PeerTube HTTP(S) traffic, including yt-dlp.
+# This is different from the reverse proxy in front of PeerTube.
+#HTTP_PROXY=http://username:[email protected]:3128
+#HTTPS_PROXY=http://username:[email protected]:3128
+
+
+# -----------------------------------------------------------------------------
+# HTTP Video Imports (Optional)
+# -----------------------------------------------------------------------------
+#PEERTUBE_IMPORT_VIDEOS_HTTP=true
+#PEERTUBE_IMPORT_VIDEOS_HTTP_YOUTUBE_DL_RELEASE_URL=https://api.github.com/repos/yt-dlp/yt-dlp/releases
+#PEERTUBE_IMPORT_VIDEOS_HTTP_YOUTUBE_DL_RELEASE_NAME=yt-dlp
+#PEERTUBE_IMPORT_VIDEOS_HTTP_FORCE_IPV4=true
+# Dedicated outbound proxy pool for HTTP imports. PeerTube randomly selects one proxy from this list.
+# JSON array. Example: ["http://user:pass@proxy-1:3128","http://user:pass@proxy-2:3128"]
+#PEERTUBE_IMPORT_VIDEOS_HTTP_PROXIES=[]
+# Enable this to pass cookies to yt-dlp. This can help when imports need a real browser session or when your server IP is limited.
+# Only enable this if you trust users allowed to trigger imports because PeerTube will import using the account from which these cookies were exported.
+# That account may be rate limited or temporarily blocked.
+# Put your Netscape-format cookies in /data/tmp-persistent/youtube-cookies.txt.
+# See https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for cookie export instructions.
+# See https://docs.joinpeertube.org/maintain/configuration#cookies-for-youtube-imports for PeerTube-specific setup.
+#PEERTUBE_IMPORT_VIDEOS_HTTP_COOKIES_ENABLED=false
+
+
# -----------------------------------------------------------------------------
# Logging (Optional)
# -----------------------------------------------------------------------------
diff --git a/support/docker/production/config/custom-environment-variables.yaml b/support/docker/production/config/custom-environment-variables.yaml
index 4155fd4f9c..098d4fe914 100644
--- a/support/docker/production/config/custom-environment-variables.yaml
+++ b/support/docker/production/config/custom-environment-variables.yaml
@@ -778,6 +778,10 @@ import:
proxies:
__name: PEERTUBE_IMPORT_VIDEOS_HTTP_PROXIES
__format: json
+ cookies:
+ enabled:
+ __name: PEERTUBE_IMPORT_VIDEOS_HTTP_COOKIES_ENABLED
+ __format: json
torrent:
enabled:
__name: PEERTUBE_IMPORT_VIDEOS_TORRENT