ActivityPub-federated video streaming platform using P2P directly in your web browser
Support byte range when throttling download
Chocobozzzc24908826a47
diff --git a/packages/tests/src/api/videos/video-download-throttle.ts b/packages/tests/src/api/videos/video-download-throttle.tsindex 15b67439a4..9c6e8eaa45 100644--- a/packages/tests/src/api/videos/video-download-throttle.ts+++ b/packages/tests/src/api/videos/video-download-throttle.ts@@ -150,4 +150,101 @@ describe('Test video download throttling', function () {await cleanupTests([ server ])})})++ describe('With byte range requests', function () {+ let server: PeerTubeServer+ let fileUrl: string+ let fullBody: Buffer++ before(async function () {+ this.timeout(120000)++ const { server: s, videoId } = await prepareServer(1, {+ // High enough to not slow down the test+ max_total_bytes_per_second: 10 * 1024 * 1024,+ max_bytes_per_ip_per_second: null+ })+ server = s++ const video = await server.videos.get({ id: videoId })+ fileUrl = video.files.find(f => f.hasVideo === true).fileDownloadUrl++ fullBody = await (await getClassicWebVideoDownload(server, videoId))()+ })++ it('Should download the full file when no Range header is sent', async function () {+ const res = await makeRawRequest({+ url: fileUrl,+ responseType: 'arraybuffer',+ expectedStatus: HttpStatusCode.OK_200+ })++ expect(res.headers['accept-ranges']).to.equal('bytes')+ expect(res.headers['content-length']).to.equal('' + fullBody.length)+ expect(res.body as Buffer).to.deep.equal(fullBody)+ })++ it('Should support a Range header with a start and an end', async function () {+ const res = await makeRawRequest({+ url: fileUrl,+ responseType: 'arraybuffer',+ range: 'bytes=0-99',+ expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206+ })++ expect(res.headers['content-range']).to.equal(`bytes 0-99/${fullBody.length}`)+ expect(res.headers['content-length']).to.equal('100')+ expect(res.body as Buffer).to.deep.equal(fullBody.subarray(0, 100))+ })++ it('Should support a Range header with only a start', async function () {+ const res = await makeRawRequest({+ url: fileUrl,+ responseType: 'arraybuffer',+ range: 'bytes=100-',+ expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206+ })++ expect(res.headers['content-range']).to.equal(`bytes 100-${fullBody.length - 1}/${fullBody.length}`)+ expect(res.body as Buffer).to.deep.equal(fullBody.subarray(100))+ })++ it('Should support a Range header with only a suffix length', async function () {+ const res = await makeRawRequest({+ url: fileUrl,+ responseType: 'arraybuffer',+ range: 'bytes=-100',+ expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206+ })++ const expectedStart = fullBody.length - 100+ expect(res.headers['content-range']).to.equal(`bytes ${expectedStart}-${fullBody.length - 1}/${fullBody.length}`)+ expect(res.body as Buffer).to.deep.equal(fullBody.subarray(expectedStart))+ })++ it('Should ignore a malformed Range header and return the full file', async function () {+ const res = await makeRawRequest({+ url: fileUrl,+ responseType: 'arraybuffer',+ range: 'bytes=abc-def',+ expectedStatus: HttpStatusCode.OK_200+ })++ expect(res.body as Buffer).to.deep.equal(fullBody)+ })++ it('Should return 416 for an out of range Range header', async function () {+ const res = await makeRawRequest({+ url: fileUrl,+ range: `bytes=${fullBody.length + 1000}-`,+ expectedStatus: HttpStatusCode.RANGE_NOT_SATISFIABLE_416+ })++ expect(res.headers['content-range']).to.equal(`bytes */${fullBody.length}`)+ })++ after(async function () {+ await cleanupTests([ server ])+ })+ })})diff --git a/server/core/controllers/download.ts b/server/core/controllers/download.tsindex c971b58d32..5a3c005866 100644--- a/server/core/controllers/download.ts+++ b/server/core/controllers/download.ts@@ -1,6 +1,7 @@import { forceNumber, maxBy } from '@peertube/peertube-core-utils'import { FileStorage, HttpStatusCode, VideoResolution, VideoStreamingPlaylistType } from '@peertube/peertube-models'import { exists } from '@server/helpers/custom-validators/misc.js'+import { parseRangeHeader } from '@server/helpers/express-utils.js'import { logger, loggerTagsFactory } from '@server/helpers/logger.js'import { generateRequestStream } from '@server/helpers/requests.js'import { ThrottleStream } from '@server/helpers/stream-throttle.js'@@ -23,6 +24,7 @@ import { create as createContentDisposition } from 'content-disposition'import cors from 'cors'import express from 'express'import { createReadStream } from 'fs'+import { stat } from 'fs/promises'import { join } from 'path'import { pipeline } from 'stream/promises'import { DOWNLOAD_PATHS, WEBSERVER } from '../initializers/constants.js'@@ -180,7 +182,7 @@ async function downloadWebVideoFile (req: express.Request, res: express.Response}await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), path => {- return downloadLocalFileWithOptionalThrottle({ res, path, downloadFilename, ip: req.ip })+ return downloadLocalFileWithOptionalThrottle({ req, res, path, downloadFilename })})}@@ -223,7 +225,7 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response}await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(streamingPlaylist), path => {- return downloadLocalFileWithOptionalThrottle({ res, path, downloadFilename, ip: req.ip })+ return downloadLocalFileWithOptionalThrottle({ req, res, path, downloadFilename })})}@@ -320,10 +322,10 @@ function downloadUserExport (req: express.Request, res: express.Response) {}return downloadLocalFileWithOptionalThrottle({+ req,res,path: getFSUserExportFilePath(userExport),- downloadFilename,- ip: req.ip+ downloadFilename})}@@ -337,10 +339,10 @@ function downloadOriginalFile (req: express.Request, res: express.Response) {}return downloadLocalFileWithOptionalThrottle({+ req,res,path: VideoPathManager.Instance.getFSOriginalVideoFilePath(videoSource.keptOriginalFilename),- downloadFilename,- ip: req.ip+ downloadFilename})}@@ -401,31 +403,68 @@ function checkAllowResult (res: express.Response, allowParameters: any, result?:}async function downloadLocalFileWithOptionalThrottle (options: {+ req: express.Requestres: express.Responsepath: stringdownloadFilename: string- ip?: string}) {- const { res, path, downloadFilename, ip } = options+ const { req, res, path, downloadFilename } = optionsconst totalBytesPerSecond = CONFIG.DOWNLOAD.MAX_TOTAL_BYTES_PER_SECONDconst bytesPerIpPerSecond = CONFIG.DOWNLOAD.MAX_BYTES_PER_IP_PER_SECONDif (!totalBytesPerSecond && !bytesPerIpPerSecond) return res.download(path, downloadFilename)+ let size: number++ try {+ const statResult = await stat(path)+ size = statResult.size+ } catch (err) {+ if (err.code === 'ENOENT') return res.sendStatus(HttpStatusCode.NOT_FOUND_404)++ throw err+ }++ const range = parseRangeHeader(req.headers.range, size)+ if (range === 'unsatisfiable') {+ res.setHeader('Content-Range', `bytes */${size}`)+ return res.sendStatus(HttpStatusCode.RANGE_NOT_SATISFIABLE_416)+ }+res.setHeader('Content-Disposition', createContentDisposition(downloadFilename))res.setHeader('Content-Type', 'application/octet-stream')+ res.setHeader('Accept-Ranges', 'bytes')++ if (range) {+ res.status(HttpStatusCode.PARTIAL_CONTENT_206)+ res.setHeader('Content-Range', `bytes ${range.start}-${range.end}/${size}`)+ res.setHeader('Content-Length', range.end - range.start + 1)+ } else {+ res.setHeader('Content-Length', size)+ }++ const readStream = createReadStream(+ path,+ range+ ? { start: range.start, end: range.end }+ : undefined+ )- const readStream = createReadStream(path)readStream.on('error', err => {if (res.headersSent) returnif ((err as any).code === 'ENOENT') return res.sendStatus(HttpStatusCode.NOT_FOUND_404)- return res.sendStatus(HttpStatusCode.INTERNAL_SERVER_ERROR_500)+ logger.error(`Cannot read local file ${path} for download`, { err, ...lTags() })++ return res.fail({+ status: HttpStatusCode.INTERNAL_SERVER_ERROR_500,+ message: err.message+ })})- await pipeline(readStream, new ThrottleStream({ totalBytesPerSecond, bytesPerIpPerSecond, ip }), res)+ await pipeline(readStream, new ThrottleStream({ totalBytesPerSecond, bytesPerIpPerSecond, ip: req.ip }), res)}async function redirectVideoDownloadToObjectStorage (options: {diff --git a/server/core/helpers/express-utils.ts b/server/core/helpers/express-utils.tsindex b902632aea..b890c2824c 100644--- a/server/core/helpers/express-utils.ts+++ b/server/core/helpers/express-utils.ts@@ -1,3 +1,4 @@+import { forceNumber } from '@peertube/peertube-core-utils'import { NSFWFlag, VideosCommonQuery } from '@peertube/peertube-models'import { getLowercaseExtension } from '@peertube/peertube-node-utils'import express, { RequestHandler } from 'express'@@ -5,10 +6,10 @@ import multer, { diskStorage } from 'multer'import { CONFIG } from '../initializers/config.js'import { REMOTE_SCHEME } from '../initializers/constants.js'import { isArray } from './custom-validators/misc.js'+import { deleteFileAndCatch } from './fs.js'import { logger } from './logger.js'import { generateRandomString } from './utils.js'import { getExtFromMimetype } from './video.js'-import { deleteFileAndCatch } from './fs.js'// ---------------------------------------------------------------------------// Extract NSFW Filters options to list videos@@ -173,6 +174,41 @@ export function getAuthUser (res: express.Response) {: undefined}+// Only supports a single "bytes=start-end" range+export function parseRangeHeader (rangeHeader: string | undefined, size: number):+ | { start: number, end: number }+ | 'unsatisfiable'+ | undefined+{+ if (!rangeHeader) return undefined++ const match = /^bytes=(\d*)-(\d*)$/.exec(rangeHeader)+ if (!match || (!match[1] && !match[2])) return undefined++ const [ , rawStart, rawEnd ] = match++ let start: number+ let end: number++ if (rawStart === '') {+ start = Math.max(size - forceNumber(rawEnd), 0)++ end = size - 1+ } else {+ start = forceNumber(rawStart)++ end = rawEnd === ''+ ? size - 1 :+ Math.min(forceNumber(rawEnd), size - 1)+ }++ if (Number.isNaN(start) || Number.isNaN(end) || start > end || start >= size) {+ return 'unsatisfiable'+ }++ return { start, end }+}+// ---------------------------------------------------------------------------// Private// ---------------------------------------------------------------------------