Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Support byte range when throttling download

Chocobozzzc24908826a47
diff --git a/packages/tests/src/api/videos/video-download-throttle.ts b/packages/tests/src/api/videos/video-download-throttle.ts
index 15b67439a4..9c6e8eaa45 100644
--- a/packages/tests/src/api/videos/video-download-throttle.ts
+++ b/packages/tests/src/api/videos/video-download-throttle.ts
@@ -150,4 +150,101 @@ describe('Test video download throttling', function () {
await cleanupTests([ server ])
})
})
+
+ describe('With byte range requests', function () {
+ let server: PeerTubeServer
+ let fileUrl: string
+ let fullBody: Buffer
+
+ before(async function () {
+ this.timeout(120000)
+
+ const { server: s, videoId } = await prepareServer(1, {
+ // High enough to not slow down the test
+ max_total_bytes_per_second: 10 * 1024 * 1024,
+ max_bytes_per_ip_per_second: null
+ })
+ server = s
+
+ const video = await server.videos.get({ id: videoId })
+ fileUrl = video.files.find(f => f.hasVideo === true).fileDownloadUrl
+
+ fullBody = await (await getClassicWebVideoDownload(server, videoId))()
+ })
+
+ it('Should download the full file when no Range header is sent', async function () {
+ const res = await makeRawRequest({
+ url: fileUrl,
+ responseType: 'arraybuffer',
+ expectedStatus: HttpStatusCode.OK_200
+ })
+
+ expect(res.headers['accept-ranges']).to.equal('bytes')
+ expect(res.headers['content-length']).to.equal('' + fullBody.length)
+ expect(res.body as Buffer).to.deep.equal(fullBody)
+ })
+
+ it('Should support a Range header with a start and an end', async function () {
+ const res = await makeRawRequest({
+ url: fileUrl,
+ responseType: 'arraybuffer',
+ range: 'bytes=0-99',
+ expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206
+ })
+
+ expect(res.headers['content-range']).to.equal(`bytes 0-99/${fullBody.length}`)
+ expect(res.headers['content-length']).to.equal('100')
+ expect(res.body as Buffer).to.deep.equal(fullBody.subarray(0, 100))
+ })
+
+ it('Should support a Range header with only a start', async function () {
+ const res = await makeRawRequest({
+ url: fileUrl,
+ responseType: 'arraybuffer',
+ range: 'bytes=100-',
+ expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206
+ })
+
+ expect(res.headers['content-range']).to.equal(`bytes 100-${fullBody.length - 1}/${fullBody.length}`)
+ expect(res.body as Buffer).to.deep.equal(fullBody.subarray(100))
+ })
+
+ it('Should support a Range header with only a suffix length', async function () {
+ const res = await makeRawRequest({
+ url: fileUrl,
+ responseType: 'arraybuffer',
+ range: 'bytes=-100',
+ expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206
+ })
+
+ const expectedStart = fullBody.length - 100
+ expect(res.headers['content-range']).to.equal(`bytes ${expectedStart}-${fullBody.length - 1}/${fullBody.length}`)
+ expect(res.body as Buffer).to.deep.equal(fullBody.subarray(expectedStart))
+ })
+
+ it('Should ignore a malformed Range header and return the full file', async function () {
+ const res = await makeRawRequest({
+ url: fileUrl,
+ responseType: 'arraybuffer',
+ range: 'bytes=abc-def',
+ expectedStatus: HttpStatusCode.OK_200
+ })
+
+ expect(res.body as Buffer).to.deep.equal(fullBody)
+ })
+
+ it('Should return 416 for an out of range Range header', async function () {
+ const res = await makeRawRequest({
+ url: fileUrl,
+ range: `bytes=${fullBody.length + 1000}-`,
+ expectedStatus: HttpStatusCode.RANGE_NOT_SATISFIABLE_416
+ })
+
+ expect(res.headers['content-range']).to.equal(`bytes */${fullBody.length}`)
+ })
+
+ after(async function () {
+ await cleanupTests([ server ])
+ })
+ })
})
diff --git a/server/core/controllers/download.ts b/server/core/controllers/download.ts
index c971b58d32..5a3c005866 100644
--- a/server/core/controllers/download.ts
+++ b/server/core/controllers/download.ts
@@ -1,6 +1,7 @@
import { forceNumber, maxBy } from '@peertube/peertube-core-utils'
import { FileStorage, HttpStatusCode, VideoResolution, VideoStreamingPlaylistType } from '@peertube/peertube-models'
import { exists } from '@server/helpers/custom-validators/misc.js'
+import { parseRangeHeader } from '@server/helpers/express-utils.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { generateRequestStream } from '@server/helpers/requests.js'
import { ThrottleStream } from '@server/helpers/stream-throttle.js'
@@ -23,6 +24,7 @@ import { create as createContentDisposition } from 'content-disposition'
import cors from 'cors'
import express from 'express'
import { createReadStream } from 'fs'
+import { stat } from 'fs/promises'
import { join } from 'path'
import { pipeline } from 'stream/promises'
import { DOWNLOAD_PATHS, WEBSERVER } from '../initializers/constants.js'
@@ -180,7 +182,7 @@ async function downloadWebVideoFile (req: express.Request, res: express.Response
}
await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), path => {
- return downloadLocalFileWithOptionalThrottle({ res, path, downloadFilename, ip: req.ip })
+ return downloadLocalFileWithOptionalThrottle({ req, res, path, downloadFilename })
})
}
@@ -223,7 +225,7 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response
}
await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(streamingPlaylist), path => {
- return downloadLocalFileWithOptionalThrottle({ res, path, downloadFilename, ip: req.ip })
+ return downloadLocalFileWithOptionalThrottle({ req, res, path, downloadFilename })
})
}
@@ -320,10 +322,10 @@ function downloadUserExport (req: express.Request, res: express.Response) {
}
return downloadLocalFileWithOptionalThrottle({
+ req,
res,
path: getFSUserExportFilePath(userExport),
- downloadFilename,
- ip: req.ip
+ downloadFilename
})
}
@@ -337,10 +339,10 @@ function downloadOriginalFile (req: express.Request, res: express.Response) {
}
return downloadLocalFileWithOptionalThrottle({
+ req,
res,
path: VideoPathManager.Instance.getFSOriginalVideoFilePath(videoSource.keptOriginalFilename),
- downloadFilename,
- ip: req.ip
+ downloadFilename
})
}
@@ -401,31 +403,68 @@ function checkAllowResult (res: express.Response, allowParameters: any, result?:
}
async function downloadLocalFileWithOptionalThrottle (options: {
+ req: express.Request
res: express.Response
path: string
downloadFilename: string
- ip?: string
}) {
- const { res, path, downloadFilename, ip } = options
+ const { req, res, path, downloadFilename } = options
const totalBytesPerSecond = CONFIG.DOWNLOAD.MAX_TOTAL_BYTES_PER_SECOND
const bytesPerIpPerSecond = CONFIG.DOWNLOAD.MAX_BYTES_PER_IP_PER_SECOND
if (!totalBytesPerSecond && !bytesPerIpPerSecond) return res.download(path, downloadFilename)
+ let size: number
+
+ try {
+ const statResult = await stat(path)
+ size = statResult.size
+ } catch (err) {
+ if (err.code === 'ENOENT') return res.sendStatus(HttpStatusCode.NOT_FOUND_404)
+
+ throw err
+ }
+
+ const range = parseRangeHeader(req.headers.range, size)
+ if (range === 'unsatisfiable') {
+ res.setHeader('Content-Range', `bytes */${size}`)
+ return res.sendStatus(HttpStatusCode.RANGE_NOT_SATISFIABLE_416)
+ }
+
res.setHeader('Content-Disposition', createContentDisposition(downloadFilename))
res.setHeader('Content-Type', 'application/octet-stream')
+ res.setHeader('Accept-Ranges', 'bytes')
+
+ if (range) {
+ res.status(HttpStatusCode.PARTIAL_CONTENT_206)
+ res.setHeader('Content-Range', `bytes ${range.start}-${range.end}/${size}`)
+ res.setHeader('Content-Length', range.end - range.start + 1)
+ } else {
+ res.setHeader('Content-Length', size)
+ }
+
+ const readStream = createReadStream(
+ path,
+ range
+ ? { start: range.start, end: range.end }
+ : undefined
+ )
- const readStream = createReadStream(path)
readStream.on('error', err => {
if (res.headersSent) return
if ((err as any).code === 'ENOENT') return res.sendStatus(HttpStatusCode.NOT_FOUND_404)
- return res.sendStatus(HttpStatusCode.INTERNAL_SERVER_ERROR_500)
+ logger.error(`Cannot read local file ${path} for download`, { err, ...lTags() })
+
+ return res.fail({
+ status: HttpStatusCode.INTERNAL_SERVER_ERROR_500,
+ message: err.message
+ })
})
- await pipeline(readStream, new ThrottleStream({ totalBytesPerSecond, bytesPerIpPerSecond, ip }), res)
+ await pipeline(readStream, new ThrottleStream({ totalBytesPerSecond, bytesPerIpPerSecond, ip: req.ip }), res)
}
async function redirectVideoDownloadToObjectStorage (options: {
diff --git a/server/core/helpers/express-utils.ts b/server/core/helpers/express-utils.ts
index b902632aea..b890c2824c 100644
--- a/server/core/helpers/express-utils.ts
+++ b/server/core/helpers/express-utils.ts
@@ -1,3 +1,4 @@
+import { forceNumber } from '@peertube/peertube-core-utils'
import { NSFWFlag, VideosCommonQuery } from '@peertube/peertube-models'
import { getLowercaseExtension } from '@peertube/peertube-node-utils'
import express, { RequestHandler } from 'express'
@@ -5,10 +6,10 @@ import multer, { diskStorage } from 'multer'
import { CONFIG } from '../initializers/config.js'
import { REMOTE_SCHEME } from '../initializers/constants.js'
import { isArray } from './custom-validators/misc.js'
+import { deleteFileAndCatch } from './fs.js'
import { logger } from './logger.js'
import { generateRandomString } from './utils.js'
import { getExtFromMimetype } from './video.js'
-import { deleteFileAndCatch } from './fs.js'
// ---------------------------------------------------------------------------
// Extract NSFW Filters options to list videos
@@ -173,6 +174,41 @@ export function getAuthUser (res: express.Response) {
: undefined
}
+// Only supports a single "bytes=start-end" range
+export function parseRangeHeader (rangeHeader: string | undefined, size: number):
+ | { start: number, end: number }
+ | 'unsatisfiable'
+ | undefined
+{
+ if (!rangeHeader) return undefined
+
+ const match = /^bytes=(\d*)-(\d*)$/.exec(rangeHeader)
+ if (!match || (!match[1] && !match[2])) return undefined
+
+ const [ , rawStart, rawEnd ] = match
+
+ let start: number
+ let end: number
+
+ if (rawStart === '') {
+ start = Math.max(size - forceNumber(rawEnd), 0)
+
+ end = size - 1
+ } else {
+ start = forceNumber(rawStart)
+
+ end = rawEnd === ''
+ ? size - 1 :
+ Math.min(forceNumber(rawEnd), size - 1)
+ }
+
+ if (Number.isNaN(start) || Number.isNaN(end) || start > end || start >= size) {
+ return 'unsatisfiable'
+ }
+
+ return { start, end }
+}
+
// ---------------------------------------------------------------------------
// Private
// ---------------------------------------------------------------------------