High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
feat(server): vaapi hardware decoding (#13561)
Mert23646f0d551d
diff --git a/docs/docs/features/hardware-transcoding.md b/docs/docs/features/hardware-transcoding.mdindex 7f74140ac0..756bb6823c 100644--- a/docs/docs/features/hardware-transcoding.md+++ b/docs/docs/features/hardware-transcoding.md@@ -23,7 +23,7 @@ You do not need to redo any transcoding jobs after enabling hardware acceleratio- Raspberry Pi is currently not supported.- Two-pass mode is only supported for NVENC. Other APIs will ignore this setting.- By default, only encoding is currently hardware accelerated. This means the CPU is still used for software decoding and tone-mapping.- - NVENC and RKMPP can be fully accelerated by enabling hardware decoding in the video transcoding settings.+ - You can benefit from end-to-end acceleration by enabling hardware decoding in the video transcoding settings.- Hardware dependent- Codec support varies, but H.264 and HEVC are usually supported.- Notably, NVIDIA and AMD GPUs do not support VP9 encoding.@@ -66,7 +66,7 @@ For RKMPP to work:3. Redeploy the `immich-server` container with these updated settings.4. In the Admin page under `Video transcoding settings`, change the hardware acceleration setting to the appropriate option and save.-5. (Optional) If using a compatible backend, you may enable hardware decoding for optimal performance.+5. (Optional) Enable hardware decoding for optimal performance.#### Single Compose Filediff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.tsindex 703794e8b7..0489169c1a 100644--- a/server/src/services/media.service.spec.ts+++ b/server/src/services/media.service.spec.ts@@ -1619,7 +1619,7 @@ describe(MediaService.name, () => {'-refs 5','-g 256','-v verbose',- '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720',+ '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720:mode=hq','-preset 7','-global_quality:v 23','-maxrate 10000k',@@ -1803,7 +1803,7 @@ describe(MediaService.name, () => {'-strict unofficial','-g 256','-v verbose',- '-vf format=nv12,hwupload,scale_vaapi=-2:720',+ '-vf format=nv12,hwupload,scale_vaapi=-2:720:mode=hq:out_range=pc','-compression_level 7','-rc_mode 1',]),@@ -1946,6 +1946,79 @@ describe(MediaService.name, () => {);});+ it('should use hardware decoding for vaapi if enabled', async () => {+ storageMock.readdir.mockResolvedValue(['renderD128']);+ mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);+ systemMock.get.mockResolvedValue({+ ffmpeg: { accel: TranscodeHWAccel.VAAPI, accelDecode: true },+ });+ assetMock.getByIds.mockResolvedValue([assetStub.video]);++ await sut.handleVideoConversion({ id: assetStub.video.id });++ expect(mediaMock.transcode).toHaveBeenCalledWith(+ '/original/path.ext',+ 'upload/encoded-video/user-id/as/se/asset-id.mp4',+ expect.objectContaining({+ inputOptions: expect.arrayContaining([+ '-hwaccel vaapi',+ '-hwaccel_output_format vaapi',+ '-noautorotate',+ '-threads 1',+ ]),+ outputOptions: expect.arrayContaining([+ expect.stringContaining('scale_vaapi=-2:720:mode=hq:out_range=pc:format=nv12'),+ ]),+ twoPass: false,+ }),+ );+ });++ it('should use hardware tone-mapping for qsv if hardware decoding is enabled and should tone map', async () => {+ storageMock.readdir.mockResolvedValue(['renderD128']);+ mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);+ systemMock.get.mockResolvedValue({+ ffmpeg: { accel: TranscodeHWAccel.VAAPI, accelDecode: true },+ });+ assetMock.getByIds.mockResolvedValue([assetStub.video]);++ await sut.handleVideoConversion({ id: assetStub.video.id });++ expect(mediaMock.transcode).toHaveBeenCalledWith(+ '/original/path.ext',+ 'upload/encoded-video/user-id/as/se/asset-id.mp4',+ expect.objectContaining({+ inputOptions: expect.arrayContaining(['-hwaccel vaapi', '-hwaccel_output_format vaapi', '-threads 1']),+ outputOptions: expect.arrayContaining([+ expect.stringContaining(+ 'hwmap=derive_device=opencl,tonemap_opencl=desat=0:format=nv12:matrix=bt709:primaries=bt709:range=pc:tonemap=hable:transfer=bt709,hwmap=derive_device=vaapi:reverse=1,format=vaapi',+ ),+ ]),+ twoPass: false,+ }),+ );+ });++ it('should use preferred device for vaapi when hardware decoding', async () => {+ storageMock.readdir.mockResolvedValue(['renderD128', 'renderD129', 'renderD130']);+ mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);+ systemMock.get.mockResolvedValue({+ ffmpeg: { accel: TranscodeHWAccel.VAAPI, accelDecode: true, preferredHwDevice: 'renderD129' },+ });+ assetMock.getByIds.mockResolvedValue([assetStub.video]);++ await sut.handleVideoConversion({ id: assetStub.video.id });+ expect(mediaMock.transcode).toHaveBeenCalledWith(+ '/original/path.ext',+ 'upload/encoded-video/user-id/as/se/asset-id.mp4',+ expect.objectContaining({+ inputOptions: expect.arrayContaining(['-hwaccel vaapi', '-hwaccel_device /dev/dri/renderD129']),+ outputOptions: expect.any(Array),+ twoPass: false,+ }),+ );+ });+it('should fallback to sw transcoding if hw transcoding fails', async () => {storageMock.readdir.mockResolvedValue(['renderD128']);mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);diff --git a/server/src/utils/media.ts b/server/src/utils/media.tsindex e58ca2f470..55f92d109a 100644--- a/server/src/utils/media.ts+++ b/server/src/utils/media.ts@@ -52,7 +52,9 @@ export class BaseConfig implements VideoCodecSWConfig {break;}case TranscodeHWAccel.VAAPI: {- handler = new VAAPIConfig(config, devices);+ handler = config.accelDecode+ ? new VaapiHwDecodeConfig(config, devices)+ : new VaapiSwDecodeConfig(config, devices);break;}case TranscodeHWAccel.RKMPP: {@@ -688,7 +690,7 @@ export class QsvSwDecodeConfig extends BaseHWConfig {const options = this.getToneMapping(videoStream);options.push('format=nv12', 'hwupload=extra_hw_frames=64');if (this.shouldScale(videoStream)) {- options.push(`scale_qsv=${this.getScaling(videoStream)}`);+ options.push(`scale_qsv=${this.getScaling(videoStream)}:mode=hq`);}return options;}@@ -811,7 +813,7 @@ export class QsvHwDecodeConfig extends QsvSwDecodeConfig {}}-export class VAAPIConfig extends BaseHWConfig {+export class VaapiSwDecodeConfig extends BaseHWConfig {getBaseInputOptions() {if (this.devices.length === 0) {throw new Error('No VAAPI device found');@@ -829,7 +831,7 @@ export class VAAPIConfig extends BaseHWConfig {const options = this.getToneMapping(videoStream);options.push('format=nv12', 'hwupload');if (this.shouldScale(videoStream)) {- options.push(`scale_vaapi=${this.getScaling(videoStream)}`);+ options.push(`scale_vaapi=${this.getScaling(videoStream)}:mode=hq:out_range=pc`);}return options;@@ -878,6 +880,76 @@ export class VAAPIConfig extends BaseHWConfig {}}+export class VaapiHwDecodeConfig extends VaapiSwDecodeConfig {+ getBaseInputOptions() {+ if (this.devices.length === 0) {+ throw new Error('No VAAPI device found');+ }++ const options = [+ '-hwaccel vaapi',+ '-hwaccel_output_format vaapi',+ '-noautorotate',+ ...this.getInputThreadOptions(),+ ];+ const hwDevice = this.getPreferredHardwareDevice();+ if (hwDevice) {+ options.push(`-hwaccel_device ${hwDevice}`);+ }++ return options;+ }++ getFilterOptions(videoStream: VideoStreamInfo) {+ const options = [];+ if (this.shouldScale(videoStream) || !this.shouldToneMap(videoStream)) {+ let scaling = `scale_vaapi=${this.getScaling(videoStream)}:mode=hq:out_range=pc`;+ if (!this.shouldToneMap(videoStream)) {+ scaling += ':format=nv12';+ }+ options.push(scaling);+ }++ options.push(...this.getToneMapping(videoStream));+ return options;+ }++ getToneMapping(videoStream: VideoStreamInfo): string[] {+ if (!this.shouldToneMap(videoStream)) {+ return [];+ }++ const colors = this.getColors();+ const tonemapOptions = [+ 'desat=0',+ 'format=nv12',+ `matrix=${colors.matrix}`,+ `primaries=${colors.primaries}`,+ 'range=pc',+ `tonemap=${this.config.tonemap}`,+ `transfer=${colors.transfer}`,+ ];++ return [+ 'hwmap=derive_device=opencl',+ `tonemap_opencl=${tonemapOptions.join(':')}`,+ 'hwmap=derive_device=vaapi:reverse=1,format=vaapi',+ ];+ }++ getInputThreadOptions() {+ return [`-threads 1`];+ }++ getColors() {+ return {+ primaries: 'bt709',+ transfer: 'bt709',+ matrix: 'bt709',+ };+ }+}+export class RkmppSwDecodeConfig extends BaseHWConfig {constructor(protected config: SystemConfigFFmpegDto,diff --git a/web/src/lib/i18n/en.json b/web/src/lib/i18n/en.jsonindex 7b60fcbd2e..730efe4edf 100644--- a/web/src/lib/i18n/en.json+++ b/web/src/lib/i18n/en.json@@ -274,7 +274,7 @@"transcoding_hardware_acceleration": "Hardware Acceleration","transcoding_hardware_acceleration_description": "Experimental; much faster, but will have lower quality at the same bitrate","transcoding_hardware_decoding": "Hardware decoding",- "transcoding_hardware_decoding_setting_description": "Applies only to NVENC, QSV and RKMPP. Enables end-to-end acceleration instead of only accelerating encoding. May not work on all videos.",+ "transcoding_hardware_decoding_setting_description": "Enables end-to-end acceleration instead of only accelerating encoding. May not work on all videos.","transcoding_hevc_codec": "HEVC codec","transcoding_max_b_frames": "Maximum B-frames","transcoding_max_b_frames_description": "Higher values improve compression efficiency, but slow down encoding. May not be compatible with hardware acceleration on older devices. 0 disables B-frames, while -1 sets this value automatically.",