High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
feat(server): add hls playlist hint (#29150)
Mert7ef2de6b535a
diff --git a/mobile/openapi/lib/api/assets_api.dart b/mobile/openapi/lib/api/assets_api.dartindex e3bed71cd3..fa49d382b9 100644--- a/mobile/openapi/lib/api/assets_api.dart+++ b/mobile/openapi/lib/api/assets_api.dart@@ -982,7 +982,9 @@ class AssetsApi {/// * [String] key:////// * [String] slug:- Future<Response> getMediaPlaylistWithHttpInfo(String id, String sessionId, int variantIndex, { String? key, String? slug, Future<void>? abortTrigger, }) async {+ ///+ /// * [num] xImmichHlsPos:+ Future<Response> getMediaPlaylistWithHttpInfo(String id, String sessionId, int variantIndex, { String? key, String? slug, num? xImmichHlsPos, Future<void>? abortTrigger, }) async {// ignore: prefer_const_declarationsfinal apiPath = r'/assets/{id}/video/stream/{sessionId}/{variantIndex}/playlist.m3u8'.replaceAll('{id}', id)@@ -1003,6 +1005,10 @@ class AssetsApi {queryParams.addAll(_queryParams('', 'slug', slug));}+ if (xImmichHlsPos != null) {+ headerParams[r'x-immich-hls-pos'] = parameterToString(xImmichHlsPos);+ }+const contentTypes = <String>[];@@ -1033,8 +1039,10 @@ class AssetsApi {/// * [String] key:////// * [String] slug:- Future<String?> getMediaPlaylist(String id, String sessionId, int variantIndex, { String? key, String? slug, Future<void>? abortTrigger, }) async {- final response = await getMediaPlaylistWithHttpInfo(id, sessionId, variantIndex, key: key, slug: slug, abortTrigger: abortTrigger,);+ ///+ /// * [num] xImmichHlsPos:+ Future<String?> getMediaPlaylist(String id, String sessionId, int variantIndex, { String? key, String? slug, num? xImmichHlsPos, Future<void>? abortTrigger, }) async {+ final response = await getMediaPlaylistWithHttpInfo(id, sessionId, variantIndex, key: key, slug: slug, xImmichHlsPos: xImmichHlsPos, abortTrigger: abortTrigger,);if (response.statusCode >= HttpStatus.badRequest) {throw ApiException(response.statusCode, await _decodeBodyBytes(response));}diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.jsonindex 199dca8d76..8df2d80e69 100644--- a/open-api/immich-openapi-specs.json+++ b/open-api/immich-openapi-specs.json@@ -4924,6 +4924,15 @@"maximum": 9007199254740991,"type": "integer"}+ },+ {+ "name": "x-immich-hls-pos",+ "required": false,+ "in": "header",+ "schema": {+ "minimum": 0,+ "type": "number"+ }}],"responses": {diff --git a/packages/sdk/src/fetch-client.ts b/packages/sdk/src/fetch-client.tsindex b7202c2a1a..e230917b74 100644--- a/packages/sdk/src/fetch-client.ts+++ b/packages/sdk/src/fetch-client.ts@@ -4452,12 +4452,13 @@ export function endSession({ id, key, sessionId, slug }: {/*** Get HLS media playlist*/-export function getMediaPlaylist({ id, key, sessionId, slug, variantIndex }: {+export function getMediaPlaylist({ id, key, sessionId, slug, variantIndex, xImmichHlsPos }: {id: string;key?: string;sessionId: string;slug?: string;variantIndex: number;+ xImmichHlsPos?: number;}, opts?: Oazapfts.RequestOpts) {return oazapfts.ok(oazapfts.fetchBlob<{status: 200;@@ -4466,7 +4467,10 @@ export function getMediaPlaylist({ id, key, sessionId, slug, variantIndex }: {key,slug}))}`, {- ...opts+ ...opts,+ headers: oazapfts.mergeHeaders(opts?.headers, {+ "x-immich-hls-pos": xImmichHlsPos+ })}));}/**diff --git a/server/src/constants.ts b/server/src/constants.tsindex 8a0f979ff0..7760fa8e8c 100644--- a/server/src/constants.ts+++ b/server/src/constants.ts@@ -223,6 +223,12 @@ export const SUPPORTED_HWA_CODECS: Record<TranscodeHardwareAcceleration, VideoCoexport const HLS_BACKPRESSURE_PAUSE_SEGMENTS = 30;export const HLS_BACKPRESSURE_RESUME_SEGMENTS = 15;export const HLS_CLEANUP_INTERVAL_MS = 60 * 1000;+export const HLS_CRF: Record<VideoCodec, number> = {+ [VideoCodec.H264]: 23,+ [VideoCodec.Hevc]: 28,+ [VideoCodec.Vp9]: 31,+ [VideoCodec.Av1]: 35,+};export const HLS_INACTIVITY_TIMEOUT_MS = 5 * 60 * 1000;export const HLS_LEASE_DURATION_MS = 30 * 60 * 1000;export const HLS_PLAYLIST_CONTENT_TYPE = 'application/vnd.apple.mpegurl';diff --git a/server/src/controllers/video-stream.controller.ts b/server/src/controllers/video-stream.controller.tsindex c5f447af88..ed99f67296 100644--- a/server/src/controllers/video-stream.controller.ts+++ b/server/src/controllers/video-stream.controller.ts@@ -6,6 +6,7 @@ import { HLS_PLAYLIST_CONTENT_TYPE } from 'src/constants';import { Endpoint, HistoryBuilder } from 'src/decorators';import { AuthDto } from 'src/dtos/auth.dto';import {+ HlsPlaylistHeaderDto,HlsSegmentHeaderDto,HlsSegmentParamDto,HlsSessionParamDto,@@ -50,8 +51,17 @@ export class VideoStreamController {description: 'Returns an HLS media playlist for one variant of the streaming session.',history: new HistoryBuilder().added('v3').alpha('v3'),})- getMediaPlaylist(@Auth() auth: AuthDto, @Param() { id, sessionId }: HlsVariantParamDto) {- return this.service.getMediaPlaylist(auth, id, sessionId);+ getMediaPlaylist(+ @Auth() auth: AuthDto,+ @Param() { id, sessionId, variantIndex }: HlsVariantParamDto,+ @Headers() headers: HlsPlaylistHeaderDto,+ ) {+ try {+ headers = HlsPlaylistHeaderDto.create(headers);+ } catch (error) {+ throw new ZodValidationException(error);+ }+ return this.service.getMediaPlaylist(auth, id, sessionId, variantIndex, headers[ImmichHeader.HlsPosition]);}@Get(':id/video/stream/:sessionId/:variantIndex/:filename')diff --git a/server/src/dtos/streaming.dto.ts b/server/src/dtos/streaming.dto.tsindex e1d98745f8..c83c3e38af 100644--- a/server/src/dtos/streaming.dto.ts+++ b/server/src/dtos/streaming.dto.ts@@ -32,3 +32,11 @@ const HlsSegmentHeaderSchema = z.object({});export class HlsSegmentHeaderDto extends createZodDto(HlsSegmentHeaderSchema) {}++const HlsPlaylistHeaderSchema = z.object({+ // Lets the client hint at which segment will be loaded after the playlist.+ // A position rather than a segment index since indices aren't comparable across variants.+ [ImmichHeader.HlsPosition]: z.coerce.number().min(0).optional(),+});++export class HlsPlaylistHeaderDto extends createZodDto(HlsPlaylistHeaderSchema) {}diff --git a/server/src/enum.ts b/server/src/enum.tsindex 45b77769c6..491beca10b 100644--- a/server/src/enum.ts+++ b/server/src/enum.ts@@ -25,6 +25,7 @@ export enum ImmichHeader {Checksum = 'x-immich-checksum',CorrelationId = 'X-Correlation-ID',HlsInitSegment = 'x-immich-hls-msn',+ HlsPosition = 'x-immich-hls-pos',}export enum ImmichQuery {diff --git a/server/src/services/hls.service.spec.ts b/server/src/services/hls.service.spec.tsindex 9d9536f848..a18f7c6d22 100644--- a/server/src/services/hls.service.spec.ts+++ b/server/src/services/hls.service.spec.ts@@ -8,6 +8,7 @@ import { newTestService, ServiceMocks } from 'test/utils';// EXTINF values come from FFmpeg's playlist to enforce an exact matchconst eiffelExpectedMediaPlaylist = `#EXTM3U#EXT-X-VERSION:7+#EXT-X-INDEPENDENT-SEGMENTS#EXT-X-TARGETDURATION:2#EXT-X-MEDIA-SEQUENCE:0#EXT-X-PLAYLIST-TYPE:VOD@@ -41,6 +42,7 @@ seg_11.m4sconst waterfallExpectedMediaPlaylist = `#EXTM3U#EXT-X-VERSION:7+#EXT-X-INDEPENDENT-SEGMENTS#EXT-X-TARGETDURATION:2#EXT-X-MEDIA-SEQUENCE:0#EXT-X-PLAYLIST-TYPE:VOD@@ -62,6 +64,7 @@ seg_5.m4sconst trainExpectedMediaPlaylist = `#EXTM3U#EXT-X-VERSION:7+#EXT-X-INDEPENDENT-SEGMENTS#EXT-X-TARGETDURATION:2#EXT-X-MEDIA-SEQUENCE:0#EXT-X-PLAYLIST-TYPE:VOD@@ -95,6 +98,7 @@ const sessionId = '00000000-0000-0000-0000-000000000000';const eiffelExpectedMasterDisabled = `#EXTM3U#EXT-X-VERSION:7+#EXT-X-INDEPENDENT-SEGMENTS#EXT-X-STREAM-INF:BANDWIDTH=1000000,RESOLUTION=480x852,CODECS="av01.0.04M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910${sessionId}/0/playlist.m3u8#EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910@@ -117,6 +121,7 @@ ${sessionId}/8/playlist.m3u8const eiffelExpectedMasterRkmpp = `#EXTM3U#EXT-X-VERSION:7+#EXT-X-INDEPENDENT-SEGMENTS#EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910${sessionId}/1/playlist.m3u8#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=480x852,CODECS="avc1.64001e,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910@@ -133,6 +138,7 @@ ${sessionId}/8/playlist.m3u8const waterfallExpectedMasterDisabled = `#EXTM3U#EXT-X-VERSION:7+#EXT-X-INDEPENDENT-SEGMENTS#EXT-X-STREAM-INF:BANDWIDTH=1000000,RESOLUTION=480x852,CODECS="av01.0.04M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830${sessionId}/0/playlist.m3u8#EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830@@ -218,12 +224,58 @@ describe(HlsService.name, () => {it.each(fixtures)('matches FFmpeg for $data.originalPath', async ({ data, playlist }) => {mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));mocks.videoStream.getForMediaPlaylist.mockResolvedValue(data);- await expect(sut.getMediaPlaylist(auth, assetId, sessionId)).resolves.toBe(playlist);+ await expect(sut.getMediaPlaylist(auth, assetId, sessionId, 0)).resolves.toBe(playlist);});it('throws NotFoundException when the session/asset cannot be loaded', async () => {mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));- await expect(sut.getMediaPlaylist(auth, assetId, sessionId)).rejects.toBeInstanceOf(NotFoundException);+ await expect(sut.getMediaPlaylist(auth, assetId, sessionId, 0)).rejects.toBeInstanceOf(NotFoundException);+ });++ it('prewarms transcoding at the segment containing the hinted position', async () => {+ mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));+ mocks.videoStream.getForMediaPlaylist.mockResolvedValue(eiffelTower);+ await sut.getMediaPlaylist(auth, assetId, sessionId, 1, 10.5);+ expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsSegmentRequest', {+ sessionId,+ assetId,+ variantIndex: 1,+ segmentIndex: 5,+ });+ });++ it('prewarms from the last requested segment when no hint is given', async () => {+ mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));+ mocks.videoStream.getSession.mockResolvedValue({ id: sessionId, assetId } as never);+ mocks.storage.checkFileExists.mockResolvedValue(true);+ await sut.getSegment(auth, assetId, sessionId, 0, 'seg_5.m4s');++ mocks.videoStream.getForMediaPlaylist.mockResolvedValue(eiffelTower);+ await sut.getMediaPlaylist(auth, assetId, sessionId, 1);+ expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsSegmentRequest', {+ sessionId,+ assetId,+ variantIndex: 1,+ segmentIndex: 6,+ });+ });++ it('does not prewarm without a hint or prior segment traffic', async () => {+ mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));+ mocks.videoStream.getForMediaPlaylist.mockResolvedValue(eiffelTower);+ await sut.getMediaPlaylist(auth, assetId, sessionId, 1);+ expect(mocks.websocket.serverSend).not.toHaveBeenCalled();+ });++ it('does not prewarm the variant the session is already playing', async () => {+ mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));+ mocks.videoStream.getSession.mockResolvedValue({ id: sessionId, assetId } as never);+ mocks.storage.checkFileExists.mockResolvedValue(true);+ await sut.getSegment(auth, assetId, sessionId, 1, 'seg_5.m4s');++ mocks.videoStream.getForMediaPlaylist.mockResolvedValue(eiffelTower);+ await sut.getMediaPlaylist(auth, assetId, sessionId, 1, 12.5);+ expect(mocks.websocket.serverSend).not.toHaveBeenCalledWith('HlsSegmentRequest', expect.anything());});});@@ -314,7 +366,7 @@ describe(HlsService.name, () => {);});- it('uses the target segment for init.mp4 when provided', async () => {+ it('uses the initSegment hint for init.mp4', async () => {await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4', 7);expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {sessionId,@@ -323,18 +375,18 @@ describe(HlsService.name, () => {});});- it('prefers the target segment over the lastRequested + 1 fallback', async () => {+ it('prefers the initSegment hint over the lastRequested + 1 fallback', async () => {await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_5.m4s'); // fallback would be 6mocks.websocket.serverSend.mockClear();- await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4', 12);+ await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4', 10);expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {sessionId,variantIndex,- segmentIndex: 12,+ segmentIndex: 10,});});- it('ignores the target segment for media segment requests (the filename wins)', async () => {+ it('ignores the initSegment hint for media segment requests (the filename wins)', async () => {await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_5.m4s', 99);expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {sessionId,diff --git a/server/src/services/hls.service.ts b/server/src/services/hls.service.tsindex 034f05bf6c..0cbb53e1cd 100644--- a/server/src/services/hls.service.ts+++ b/server/src/services/hls.service.ts@@ -21,6 +21,7 @@ import { ImmichFileResponse } from 'src/utils/file';import { getOutputSize } from 'src/utils/media';type AssetWithStreamInfo = { videoStream: VideoStreamInfo & { timeBase: number }; packets: VideoPacketInfo };+type Segmentation = { fps: number; framesPerSegment: number; segmentCount: number; segmentDuration: number };type ApiSession = { lastRequestedSegment: number | null; lastVariantIndex: number | null };@Injectable()@@ -71,7 +72,7 @@ export class HlsService extends BaseService {return this.generateMainPlaylist(sessionId, ffmpeg, asset);}- async getMediaPlaylist(auth: AuthDto, assetId: string, sessionId: string) {+ async getMediaPlaylist(auth: AuthDto, assetId: string, sessionId: string, variantIndex: number, position?: number) {await this.requireAccess({ auth, permission: Permission.AssetView, ids: [assetId] });const asset = await this.videoStreamRepository.getForMediaPlaylist(assetId, sessionId);@@ -79,7 +80,11 @@ export class HlsService extends BaseService {throw new NotFoundException('Asset not found or metadata not yet ready for streaming');}- return this.generateMediaPlaylist(asset);+ const segmentation = this.getSegmentation(asset);+ const hintedSegment = position === undefined ? undefined : this.positionToSegmentIndex(segmentation, position);+ this.prewarmVariant(assetId, sessionId, variantIndex, hintedSegment);++ return this.generateMediaPlaylist(asset, segmentation);}async getSegment(@@ -129,7 +134,7 @@ export class HlsService extends BaseService {const fps = ((asset.packets.packetCount * asset.videoStream.timeBase) / asset.packets.totalDuration).toFixed(3);const sourceResolution = Math.min(asset.videoStream.height, asset.videoStream.width);const targetResolution = Math.max(sourceResolution, HLS_VARIANTS[0].resolution);- const lines = ['#EXTM3U', `#EXT-X-VERSION:${HLS_VERSION}`];+ const lines = ['#EXTM3U', `#EXT-X-VERSION:${HLS_VERSION}`, '#EXT-X-INDEPENDENT-SEGMENTS'];for (let i = 0; i < HLS_VARIANTS.length; i++) {const { resolution, bitrate, codec, codecString } = HLS_VARIANTS[i];if (resolution > targetResolution || !SUPPORTED_HWA_CODECS[ffmpeg.accel].includes(codec)) {@@ -143,24 +148,33 @@ export class HlsService extends BaseService {}lines.push('');- if (lines.length === 3) {+ if (lines.length === 4) {throw new NotFoundException('No supported variants for this video');}return lines.join('\n');}- private generateMediaPlaylist({ videoStream, packets }: AssetWithStreamInfo) {+ private getSegmentation({ videoStream, packets }: AssetWithStreamInfo): Segmentation {const fps = (packets.packetCount * videoStream.timeBase) / packets.totalDuration;const framesPerSegment = Math.ceil(HLS_SEGMENT_DURATION * fps);- const fullSegmentDuration = framesPerSegment / fps;const segmentCount = Math.ceil(packets.outputFrames / framesPerSegment);+ return { fps, framesPerSegment, segmentCount, segmentDuration: framesPerSegment / fps };+ }++ private positionToSegmentIndex({ segmentDuration, segmentCount }: Segmentation, position: number) {+ return Math.min(Math.max(Math.floor(position / segmentDuration), 0), segmentCount - 1);+ }++ private generateMediaPlaylist({ packets }: AssetWithStreamInfo, segmentation: Segmentation) {+ const { fps, framesPerSegment, segmentCount, segmentDuration: fullSegmentDuration } = segmentation;const lastSegmentFrames = packets.outputFrames - framesPerSegment * (segmentCount - 1);const lastSegmentDuration = lastSegmentFrames / fps;const lines = ['#EXTM3U',`#EXT-X-VERSION:${HLS_VERSION}`,+ '#EXT-X-INDEPENDENT-SEGMENTS',`#EXT-X-TARGETDURATION:${HLS_SEGMENT_DURATION}`,'#EXT-X-MEDIA-SEQUENCE:0','#EXT-X-PLAYLIST-TYPE:VOD',@@ -175,6 +189,19 @@ export class HlsService extends BaseService {return lines.join('\n');}+ private prewarmVariant(assetId: string, sessionId: string, variantIndex: number, hintedSegment?: number) {+ const session = this.sessions.get(sessionId);+ if (session?.lastVariantIndex === variantIndex) {+ return;+ }++ const nextSegment = session && session.lastRequestedSegment !== null ? session.lastRequestedSegment + 1 : undefined;+ const segmentIndex = hintedSegment ?? nextSegment;+ if (segmentIndex !== undefined) {+ this.websocketRepository.serverSend('HlsSegmentRequest', { sessionId, assetId, variantIndex, segmentIndex });+ }+ }+private getSegmentKey({ sessionId, variantIndex, segmentIndex }: ArgOf<'HlsSegmentResult'>) {return `${sessionId}:${variantIndex}:${segmentIndex}`;}diff --git a/server/src/services/transcoding.service.spec.ts b/server/src/services/transcoding.service.spec.tsindex 349cba6a7d..8909a43342 100644--- a/server/src/services/transcoding.service.spec.ts+++ b/server/src/services/transcoding.service.spec.ts@@ -381,8 +381,6 @@ describe(TranscodingService.name, () => {'50','-keyint_min','50',- '-crf',- '23','-copyts','-r','50130000/2012441',@@ -417,6 +415,8 @@ describe(TranscodingService.name, () => {'aac','-preset','12',+ '-crf',+ '35','-svtav1-params','hierarchical-levels=3:lookahead=0:enable-tf=0:mbr=4000k','-hls_segment_filename',@@ -436,6 +436,8 @@ describe(TranscodingService.name, () => {'hvc1','-preset','ultrafast',+ '-crf',+ '28','-maxrate','2500k','-bufsize',@@ -459,6 +461,8 @@ describe(TranscodingService.name, () => {'aac','-preset','ultrafast',+ '-crf',+ '23','-maxrate','2500k','-bufsize',diff --git a/server/src/services/transcoding.service.ts b/server/src/services/transcoding.service.tsindex dae14f6257..c79eec7884 100644--- a/server/src/services/transcoding.service.ts+++ b/server/src/services/transcoding.service.ts@@ -5,6 +5,7 @@ import {HLS_BACKPRESSURE_PAUSE_SEGMENTS,HLS_BACKPRESSURE_RESUME_SEGMENTS,HLS_CLEANUP_INTERVAL_MS,+ HLS_CRF,HLS_INACTIVITY_TIMEOUT_MS,HLS_LEASE_DURATION_MS,HLS_SEGMENT_DURATION,@@ -221,6 +222,7 @@ export class TranscodingService extends BaseService {targetResolution: String(variant.resolution),maxBitrate: `${Math.round(variant.bitrate / 1000)}k`,gopSize: gop,+ crf: HLS_CRF[variant.codec],},this.videoInterfaces,{ strictGop: true, lowLatency: true },