Skip to content

High performance self-hosted photo and video management solution.

feat(server): vaapi hardware decoding (#13561)

Mert23646f0d551d
diff --git a/docs/docs/features/hardware-transcoding.md b/docs/docs/features/hardware-transcoding.md
index 7f74140ac0..756bb6823c 100644
--- a/docs/docs/features/hardware-transcoding.md
+++ b/docs/docs/features/hardware-transcoding.md
@@ -23,7 +23,7 @@ You do not need to redo any transcoding jobs after enabling hardware acceleratio
- Raspberry Pi is currently not supported.
- Two-pass mode is only supported for NVENC. Other APIs will ignore this setting.
- By default, only encoding is currently hardware accelerated. This means the CPU is still used for software decoding and tone-mapping.
- - NVENC and RKMPP can be fully accelerated by enabling hardware decoding in the video transcoding settings.
+ - You can benefit from end-to-end acceleration by enabling hardware decoding in the video transcoding settings.
- Hardware dependent
- Codec support varies, but H.264 and HEVC are usually supported.
- Notably, NVIDIA and AMD GPUs do not support VP9 encoding.
@@ -66,7 +66,7 @@ For RKMPP to work:
3. Redeploy the `immich-server` container with these updated settings.
4. In the Admin page under `Video transcoding settings`, change the hardware acceleration setting to the appropriate option and save.
-5. (Optional) If using a compatible backend, you may enable hardware decoding for optimal performance.
+5. (Optional) Enable hardware decoding for optimal performance.
#### Single Compose File
diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts
index 703794e8b7..0489169c1a 100644
--- a/server/src/services/media.service.spec.ts
+++ b/server/src/services/media.service.spec.ts
@@ -1619,7 +1619,7 @@ describe(MediaService.name, () => {
'-refs 5',
'-g 256',
'-v verbose',
- '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720',
+ '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720:mode=hq',
'-preset 7',
'-global_quality:v 23',
'-maxrate 10000k',
@@ -1803,7 +1803,7 @@ describe(MediaService.name, () => {
'-strict unofficial',
'-g 256',
'-v verbose',
- '-vf format=nv12,hwupload,scale_vaapi=-2:720',
+ '-vf format=nv12,hwupload,scale_vaapi=-2:720:mode=hq:out_range=pc',
'-compression_level 7',
'-rc_mode 1',
]),
@@ -1946,6 +1946,79 @@ describe(MediaService.name, () => {
);
});
+ it('should use hardware decoding for vaapi if enabled', async () => {
+ storageMock.readdir.mockResolvedValue(['renderD128']);
+ mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+ systemMock.get.mockResolvedValue({
+ ffmpeg: { accel: TranscodeHWAccel.VAAPI, accelDecode: true },
+ });
+ assetMock.getByIds.mockResolvedValue([assetStub.video]);
+
+ await sut.handleVideoConversion({ id: assetStub.video.id });
+
+ expect(mediaMock.transcode).toHaveBeenCalledWith(
+ '/original/path.ext',
+ 'upload/encoded-video/user-id/as/se/asset-id.mp4',
+ expect.objectContaining({
+ inputOptions: expect.arrayContaining([
+ '-hwaccel vaapi',
+ '-hwaccel_output_format vaapi',
+ '-noautorotate',
+ '-threads 1',
+ ]),
+ outputOptions: expect.arrayContaining([
+ expect.stringContaining('scale_vaapi=-2:720:mode=hq:out_range=pc:format=nv12'),
+ ]),
+ twoPass: false,
+ }),
+ );
+ });
+
+ it('should use hardware tone-mapping for qsv if hardware decoding is enabled and should tone map', async () => {
+ storageMock.readdir.mockResolvedValue(['renderD128']);
+ mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
+ systemMock.get.mockResolvedValue({
+ ffmpeg: { accel: TranscodeHWAccel.VAAPI, accelDecode: true },
+ });
+ assetMock.getByIds.mockResolvedValue([assetStub.video]);
+
+ await sut.handleVideoConversion({ id: assetStub.video.id });
+
+ expect(mediaMock.transcode).toHaveBeenCalledWith(
+ '/original/path.ext',
+ 'upload/encoded-video/user-id/as/se/asset-id.mp4',
+ expect.objectContaining({
+ inputOptions: expect.arrayContaining(['-hwaccel vaapi', '-hwaccel_output_format vaapi', '-threads 1']),
+ outputOptions: expect.arrayContaining([
+ expect.stringContaining(
+ 'hwmap=derive_device=opencl,tonemap_opencl=desat=0:format=nv12:matrix=bt709:primaries=bt709:range=pc:tonemap=hable:transfer=bt709,hwmap=derive_device=vaapi:reverse=1,format=vaapi',
+ ),
+ ]),
+ twoPass: false,
+ }),
+ );
+ });
+
+ it('should use preferred device for vaapi when hardware decoding', async () => {
+ storageMock.readdir.mockResolvedValue(['renderD128', 'renderD129', 'renderD130']);
+ mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
+ systemMock.get.mockResolvedValue({
+ ffmpeg: { accel: TranscodeHWAccel.VAAPI, accelDecode: true, preferredHwDevice: 'renderD129' },
+ });
+ assetMock.getByIds.mockResolvedValue([assetStub.video]);
+
+ await sut.handleVideoConversion({ id: assetStub.video.id });
+ expect(mediaMock.transcode).toHaveBeenCalledWith(
+ '/original/path.ext',
+ 'upload/encoded-video/user-id/as/se/asset-id.mp4',
+ expect.objectContaining({
+ inputOptions: expect.arrayContaining(['-hwaccel vaapi', '-hwaccel_device /dev/dri/renderD129']),
+ outputOptions: expect.any(Array),
+ twoPass: false,
+ }),
+ );
+ });
+
it('should fallback to sw transcoding if hw transcoding fails', async () => {
storageMock.readdir.mockResolvedValue(['renderD128']);
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts
index e58ca2f470..55f92d109a 100644
--- a/server/src/utils/media.ts
+++ b/server/src/utils/media.ts
@@ -52,7 +52,9 @@ export class BaseConfig implements VideoCodecSWConfig {
break;
}
case TranscodeHWAccel.VAAPI: {
- handler = new VAAPIConfig(config, devices);
+ handler = config.accelDecode
+ ? new VaapiHwDecodeConfig(config, devices)
+ : new VaapiSwDecodeConfig(config, devices);
break;
}
case TranscodeHWAccel.RKMPP: {
@@ -688,7 +690,7 @@ export class QsvSwDecodeConfig extends BaseHWConfig {
const options = this.getToneMapping(videoStream);
options.push('format=nv12', 'hwupload=extra_hw_frames=64');
if (this.shouldScale(videoStream)) {
- options.push(`scale_qsv=${this.getScaling(videoStream)}`);
+ options.push(`scale_qsv=${this.getScaling(videoStream)}:mode=hq`);
}
return options;
}
@@ -811,7 +813,7 @@ export class QsvHwDecodeConfig extends QsvSwDecodeConfig {
}
}
-export class VAAPIConfig extends BaseHWConfig {
+export class VaapiSwDecodeConfig extends BaseHWConfig {
getBaseInputOptions() {
if (this.devices.length === 0) {
throw new Error('No VAAPI device found');
@@ -829,7 +831,7 @@ export class VAAPIConfig extends BaseHWConfig {
const options = this.getToneMapping(videoStream);
options.push('format=nv12', 'hwupload');
if (this.shouldScale(videoStream)) {
- options.push(`scale_vaapi=${this.getScaling(videoStream)}`);
+ options.push(`scale_vaapi=${this.getScaling(videoStream)}:mode=hq:out_range=pc`);
}
return options;
@@ -878,6 +880,76 @@ export class VAAPIConfig extends BaseHWConfig {
}
}
+export class VaapiHwDecodeConfig extends VaapiSwDecodeConfig {
+ getBaseInputOptions() {
+ if (this.devices.length === 0) {
+ throw new Error('No VAAPI device found');
+ }
+
+ const options = [
+ '-hwaccel vaapi',
+ '-hwaccel_output_format vaapi',
+ '-noautorotate',
+ ...this.getInputThreadOptions(),
+ ];
+ const hwDevice = this.getPreferredHardwareDevice();
+ if (hwDevice) {
+ options.push(`-hwaccel_device ${hwDevice}`);
+ }
+
+ return options;
+ }
+
+ getFilterOptions(videoStream: VideoStreamInfo) {
+ const options = [];
+ if (this.shouldScale(videoStream) || !this.shouldToneMap(videoStream)) {
+ let scaling = `scale_vaapi=${this.getScaling(videoStream)}:mode=hq:out_range=pc`;
+ if (!this.shouldToneMap(videoStream)) {
+ scaling += ':format=nv12';
+ }
+ options.push(scaling);
+ }
+
+ options.push(...this.getToneMapping(videoStream));
+ return options;
+ }
+
+ getToneMapping(videoStream: VideoStreamInfo): string[] {
+ if (!this.shouldToneMap(videoStream)) {
+ return [];
+ }
+
+ const colors = this.getColors();
+ const tonemapOptions = [
+ 'desat=0',
+ 'format=nv12',
+ `matrix=${colors.matrix}`,
+ `primaries=${colors.primaries}`,
+ 'range=pc',
+ `tonemap=${this.config.tonemap}`,
+ `transfer=${colors.transfer}`,
+ ];
+
+ return [
+ 'hwmap=derive_device=opencl',
+ `tonemap_opencl=${tonemapOptions.join(':')}`,
+ 'hwmap=derive_device=vaapi:reverse=1,format=vaapi',
+ ];
+ }
+
+ getInputThreadOptions() {
+ return [`-threads 1`];
+ }
+
+ getColors() {
+ return {
+ primaries: 'bt709',
+ transfer: 'bt709',
+ matrix: 'bt709',
+ };
+ }
+}
+
export class RkmppSwDecodeConfig extends BaseHWConfig {
constructor(
protected config: SystemConfigFFmpegDto,
diff --git a/web/src/lib/i18n/en.json b/web/src/lib/i18n/en.json
index 7b60fcbd2e..730efe4edf 100644
--- a/web/src/lib/i18n/en.json
+++ b/web/src/lib/i18n/en.json
@@ -274,7 +274,7 @@
"transcoding_hardware_acceleration": "Hardware Acceleration",
"transcoding_hardware_acceleration_description": "Experimental; much faster, but will have lower quality at the same bitrate",
"transcoding_hardware_decoding": "Hardware decoding",
- "transcoding_hardware_decoding_setting_description": "Applies only to NVENC, QSV and RKMPP. Enables end-to-end acceleration instead of only accelerating encoding. May not work on all videos.",
+ "transcoding_hardware_decoding_setting_description": "Enables end-to-end acceleration instead of only accelerating encoding. May not work on all videos.",
"transcoding_hevc_codec": "HEVC codec",
"transcoding_max_b_frames": "Maximum B-frames",
"transcoding_max_b_frames_description": "Higher values improve compression efficiency, but slow down encoding. May not be compatible with hardware acceleration on older devices. 0 disables B-frames, while -1 sets this value automatically.",