ActivityPub-federated video streaming platform using P2P directly in your web browser
Add cookies to yt-dlp config and docker env (#7510)
Viny Brun Kriesangfb46151616a1
diff --git a/config/default.yaml b/config/default.yamlindex b33871bdf8..db2c208b4f 100644--- a/config/default.yaml+++ b/config/default.yaml@@ -884,6 +884,15 @@ import:proxies:# - "https://username:[email protected]:8888"+ cookies:+ # Enable this to pass cookies to yt-dlp. This can help when imports need a real browser session or when your server IP is limited+ # Only enable this if you trust users allowed to trigger imports because PeerTube will import using the account from which these cookies were exported+ # That account may be rate limited or temporarily blocked+ # PeerTube expects a Netscape-format file at storage/tmp-persistent/youtube-cookies.txt+ # See https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for cookie export instructions+ # See https://docs.joinpeertube.org/maintain/configuration#cookies-for-youtube-imports for PeerTube-specific setup+ enabled: false+# Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)torrent:# We recommend to only enable magnet URI/torrent import if you trust your usersdiff --git a/config/production.yaml.example b/config/production.yaml.exampleindex c37b64e2da..92655d9146 100644--- a/config/production.yaml.example+++ b/config/production.yaml.example@@ -894,6 +894,15 @@ import:proxies:# - "https://username:[email protected]:8888"+ cookies:+ # Enable this to pass cookies to yt-dlp. This can help when imports need a real browser session or when your server IP is limited+ # Only enable this if you trust users allowed to trigger imports because PeerTube will import using the account from which these cookies were exported+ # That account may be rate limited or temporarily blocked+ # PeerTube expects a Netscape-format file at /var/www/peertube/storage/tmp-persistent/youtube-cookies.txt+ # See https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for cookie export instructions+ # See https://docs.joinpeertube.org/maintain/configuration#cookies-for-youtube-imports for PeerTube-specific setup+ enabled: false+# Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)torrent:# We recommend to only enable magnet URI/torrent import if you trust your usersdiff --git a/packages/tests/src/server-helpers/youtube-dl.ts b/packages/tests/src/server-helpers/youtube-dl.tsindex f4922d5e70..25451f8364 100644--- a/packages/tests/src/server-helpers/youtube-dl.ts+++ b/packages/tests/src/server-helpers/youtube-dl.ts@@ -1,7 +1,11 @@/* oxlint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */import { expect } from 'chai'+import { mkdtemp, rm, writeFile } from 'fs/promises'+import { tmpdir } from 'os'+import { join } from 'path'import { YoutubeDLCLI } from '@peertube/peertube-server/core/helpers/youtube-dl/youtube-dl-cli.js'+import { logger } from '@peertube/peertube-server/core/helpers/logger.js'import { CONFIG } from '@peertube/peertube-server/core/initializers/config.js'describe('YoutubeDLCLI', function () {@@ -91,4 +95,96 @@ describe('YoutubeDLCLI', function () {}})})++ describe('wrapWithCookiesOptions', function () {+ let cli: any++ before(function () {+ cli = Object.create(YoutubeDLCLI.prototype)+ })++ it('Should prepend cookies file when configured and the file exists', async function () {+ const originalTmpPersistentDirDescriptor = Object.getOwnPropertyDescriptor(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR')+ const originalCookiesEnabledDescriptor = Object.getOwnPropertyDescriptor(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED')+ const tempDir = await mkdtemp(join(tmpdir(), 'peertube-cookies-'))+ const cookiesFile = join(tempDir, 'youtube-cookies.txt')++ await writeFile(cookiesFile, '# Netscape HTTP Cookie File\n')++ Object.defineProperty(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR', {+ get: () => tempDir,+ configurable: true+ })++ Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', {+ get: () => true,+ configurable: true+ })++ try {+ const inputArgs = [ '--dump-json', '-f', 'best' ]+ const result: string[] = await cli.wrapWithCookiesOptions(inputArgs)++ expect(result).to.deep.equal([ '--cookies', cookiesFile, ...inputArgs ])+ } finally {+ Object.defineProperty(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR', originalTmpPersistentDirDescriptor)+ Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', originalCookiesEnabledDescriptor)+ await rm(tempDir, { recursive: true, force: true })+ }+ })++ it('Should log an error and continue when the cookies file is missing', async function () {+ const originalTmpPersistentDirDescriptor = Object.getOwnPropertyDescriptor(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR')+ const originalCookiesEnabledDescriptor = Object.getOwnPropertyDescriptor(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED')+ const originalLoggerError = logger.error+ const tempDir = await mkdtemp(join(tmpdir(), 'peertube-cookies-'))+ const loggedMessages: any[][] = []++ Object.defineProperty(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR', {+ get: () => tempDir,+ configurable: true+ })++ Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', {+ get: () => true,+ configurable: true+ })++ ;(logger as any).error = (...args: any[]) => {+ loggedMessages.push(args)+ }++ try {+ const inputArgs = [ '--dump-json', '-f', 'best' ]+ const result: string[] = await cli.wrapWithCookiesOptions(inputArgs)++ expect(result).to.deep.equal(inputArgs)+ expect(loggedMessages).to.have.lengthOf(1)+ expect(loggedMessages[0][0]).to.contain('yt-dlp cookies are enabled but the cookies file %s does not exist')+ } finally {+ Object.defineProperty(CONFIG.STORAGE, 'TMP_PERSISTENT_DIR', originalTmpPersistentDirDescriptor)+ Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', originalCookiesEnabledDescriptor)+ ;(logger as any).error = originalLoggerError+ await rm(tempDir, { recursive: true, force: true })+ }+ })++ it('Should not modify args when cookies are disabled', async function () {+ const originalCookiesEnabledDescriptor = Object.getOwnPropertyDescriptor(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED')++ Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', {+ get: () => false,+ configurable: true+ })++ try {+ const inputArgs = [ '--dump-json', '-f', 'best' ]+ const result: string[] = await cli.wrapWithCookiesOptions(inputArgs)++ expect(result).to.deep.equal(inputArgs)+ } finally {+ Object.defineProperty(CONFIG.IMPORT.VIDEOS.HTTP.COOKIES, 'ENABLED', originalCookiesEnabledDescriptor)+ }+ })+ })})diff --git a/server/core/helpers/youtube-dl/youtube-dl-cli.ts b/server/core/helpers/youtube-dl/youtube-dl-cli.tsindex 299a3be988..3a6e2f7251 100644--- a/server/core/helpers/youtube-dl/youtube-dl-cli.ts+++ b/server/core/helpers/youtube-dl/youtube-dl-cli.ts@@ -215,6 +215,7 @@ export class YoutubeDLCLI {let completeArgs = this.wrapWithJSRuntimeOptions(args)completeArgs = this.wrapWithProxyOptions(completeArgs)+ completeArgs = await this.wrapWithCookiesOptions(completeArgs)completeArgs = this.wrapWithIPOptions(completeArgs)completeArgs = this.wrapWithFFmpegOptions(completeArgs)@@ -272,6 +273,28 @@ export class YoutubeDLCLI {return args}+ private async wrapWithCookiesOptions (args: string[]) {+ if (!CONFIG.IMPORT.VIDEOS.HTTP.COOKIES.ENABLED) {+ return args+ }++ const cookiesPath = join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, 'youtube-cookies.txt')++ if (!await pathExists(cookiesPath)) {+ logger.error(+ 'yt-dlp cookies are enabled but the cookies file %s does not exist. Continuing without cookies.',+ cookiesPath,+ lTags()+ )++ return args+ }++ logger.debug('Using cookies file %s for YoutubeDL', cookiesPath, lTags())++ return [ '--cookies', cookiesPath ].concat(args)+ }+private wrapWithFFmpegOptions (args: string[]) {if (process.env.FFMPEG_PATH) {logger.debug('Using ffmpeg location %s for YoutubeDL', process.env.FFMPEG_PATH, lTags())diff --git a/server/core/initializers/checker-before-init.ts b/server/core/initializers/checker-before-init.tsindex 54e338e106..b8c45fe279 100644--- a/server/core/initializers/checker-before-init.ts+++ b/server/core/initializers/checker-before-init.ts@@ -119,6 +119,7 @@ export function checkMissedConfig () {'import.videos.timeout','import.videos.http.force_ipv4','import.videos.http.proxies',+ 'import.videos.http.cookies.enabled','import.video_channel_synchronization.enabled','import.video_channel_synchronization.max_per_user','import.video_channel_synchronization.check_interval',diff --git a/server/core/initializers/config.ts b/server/core/initializers/config.tsindex 49fecd9c1c..e32b5524f7 100644--- a/server/core/initializers/config.ts+++ b/server/core/initializers/config.ts@@ -909,6 +909,12 @@ const CONFIG = {get PROXIES () {return config.get<string[]>('import.videos.http.proxies')+ },++ COOKIES: {+ get ENABLED () {+ return config.get<boolean>('import.videos.http.cookies.enabled')+ }}},TORRENT: {diff --git a/support/docker/production/.env b/support/docker/production/.envindex c1b872ee48..208280ead8 100644--- a/support/docker/production/.env+++ b/support/docker/production/.env@@ -102,6 +102,34 @@ PEERTUBE_OBJECT_STORAGE_UPLOAD_ACL_PUBLIC="public-read"PEERTUBE_OBJECT_STORAGE_UPLOAD_ACL_PRIVATE="private"+# -----------------------------------------------------------------------------+# Outbound Proxies (Optional)+# -----------------------------------------------------------------------------+# These proxies affect all outbound PeerTube HTTP(S) traffic, including yt-dlp.+# This is different from the reverse proxy in front of PeerTube.+#HTTP_PROXY=http://username:[email protected]:3128+#HTTPS_PROXY=http://username:[email protected]:3128+++# -----------------------------------------------------------------------------+# HTTP Video Imports (Optional)+# -----------------------------------------------------------------------------+#PEERTUBE_IMPORT_VIDEOS_HTTP=true+#PEERTUBE_IMPORT_VIDEOS_HTTP_YOUTUBE_DL_RELEASE_URL=https://api.github.com/repos/yt-dlp/yt-dlp/releases+#PEERTUBE_IMPORT_VIDEOS_HTTP_YOUTUBE_DL_RELEASE_NAME=yt-dlp+#PEERTUBE_IMPORT_VIDEOS_HTTP_FORCE_IPV4=true+# Dedicated outbound proxy pool for HTTP imports. PeerTube randomly selects one proxy from this list.+# JSON array. Example: ["http://user:pass@proxy-1:3128","http://user:pass@proxy-2:3128"]+#PEERTUBE_IMPORT_VIDEOS_HTTP_PROXIES=[]+# Enable this to pass cookies to yt-dlp. This can help when imports need a real browser session or when your server IP is limited.+# Only enable this if you trust users allowed to trigger imports because PeerTube will import using the account from which these cookies were exported.+# That account may be rate limited or temporarily blocked.+# Put your Netscape-format cookies in /data/tmp-persistent/youtube-cookies.txt.+# See https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for cookie export instructions.+# See https://docs.joinpeertube.org/maintain/configuration#cookies-for-youtube-imports for PeerTube-specific setup.+#PEERTUBE_IMPORT_VIDEOS_HTTP_COOKIES_ENABLED=false++# -----------------------------------------------------------------------------# Logging (Optional)# -----------------------------------------------------------------------------diff --git a/support/docker/production/config/custom-environment-variables.yaml b/support/docker/production/config/custom-environment-variables.yamlindex 4155fd4f9c..098d4fe914 100644--- a/support/docker/production/config/custom-environment-variables.yaml+++ b/support/docker/production/config/custom-environment-variables.yaml@@ -778,6 +778,10 @@ import:proxies:__name: PEERTUBE_IMPORT_VIDEOS_HTTP_PROXIES__format: json+ cookies:+ enabled:+ __name: PEERTUBE_IMPORT_VIDEOS_HTTP_COOKIES_ENABLED+ __format: jsontorrent:enabled:__name: PEERTUBE_IMPORT_VIDEOS_TORRENT