Skip to content

High performance self-hosted photo and video management solution.

fix(server): incorrect scaling of rotated videos when transcoding (#31473)

fabxyzd1faeb8199f9
diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts
index faed3b5aa0..950e1569bd 100644
--- a/server/src/services/media.service.spec.ts
+++ b/server/src/services/media.service.spec.ts
@@ -2182,6 +2182,45 @@ describe(MediaService.name, () => {
);
});
+ it('should scale horizontally when a vertical video is rotated to landscape', async () => {
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({
+ ...asset,
+ ...probeStub.videoStreamRotatedHorizontal2160p,
+ });
+ mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.Optimal } });
+ await sut.handleVideoConversion({ id: 'video-id' });
+ expect(mocks.media.transcode).toHaveBeenCalledWith(
+ '/original/path.ext',
+ expect.any(String),
+ expect.objectContaining({
+ inputOptions: expect.any(Array),
+ outputOptions: expect.arrayContaining([expect.stringMatching(/scale(_.+)?=-2:720/)]),
+ twoPass: false,
+ }),
+ );
+ });
+
+ it.each([
+ { accel: TranscodeHardwareAcceleration.Nvenc, scaling: 'scale_cuda=-2:720' },
+ { accel: TranscodeHardwareAcceleration.Qsv, scaling: 'scale_qsv=-1:720' },
+ { accel: TranscodeHardwareAcceleration.Vaapi, scaling: 'scale_vaapi=-2:720' },
+ { accel: TranscodeHardwareAcceleration.Rkmpp, scaling: 'scale_rkrga=-2:720' },
+ ])('should scale rotated video by its stored dimensions when decoding with $accel', async ({ accel, scaling }) => {
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamVertical2160p });
+ mocks.systemMetadata.get.mockResolvedValue({
+ ffmpeg: { accel, accelDecode: true, transcode: TranscodePolicy.Optimal },
+ });
+ await sut.handleVideoConversion({ id: 'video-id' });
+ expect(mocks.media.transcode).toHaveBeenCalledWith(
+ '/original/path.ext',
+ expect.any(String),
+ expect.objectContaining({
+ inputOptions: expect.arrayContaining(['-noautorotate']),
+ outputOptions: expect.arrayContaining([expect.stringContaining(scaling)]),
+ }),
+ );
+ });
+
it('should always scale video if height is uneven', async () => {
mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamOddHeight });
mocks.systemMetadata.get.mockResolvedValue({
diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts
index 77c9d11578..0d92aeb659 100644
--- a/server/src/utils/media.ts
+++ b/server/src/utils/media.ts
@@ -31,8 +31,9 @@ import {
export const isVideoRotated = (videoStream: VideoStreamInfo): boolean => Math.abs(videoStream.rotation) === 90;
+// whether the video is portrait once its rotation is applied
export const isVideoVertical = (videoStream: VideoStreamInfo): boolean =>
- videoStream.height > videoStream.width || isVideoRotated(videoStream);
+ videoStream.height > videoStream.width !== isVideoRotated(videoStream);
export const getOutputSize = (videoStream: VideoStreamInfo, targetRes: number) => {
const factor = Math.max(videoStream.height, videoStream.width) / Math.min(videoStream.height, videoStream.width);
@@ -364,7 +365,12 @@ export class BaseConfig implements VideoCodecSWConfig {
getScaling(videoStream: VideoStreamInfo, mult = 2) {
const targetResolution = this.getTargetResolution(videoStream);
- return isVideoVertical(videoStream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`;
+ return this.isFrameVertical(videoStream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`;
+ }
+
+ // frames reach the filters already rotated, unless decoding with -noautorotate
+ isFrameVertical(videoStream: VideoStreamInfo) {
+ return isVideoVertical(videoStream);
}
isBitrateConstrained() {
@@ -741,6 +747,11 @@ export class NvencHwDecodeConfig extends NvencSwDecodeConfig {
return ['-hwaccel', 'cuda', '-hwaccel_output_format', 'cuda', '-noautorotate', ...this.getInputThreadOptions()];
}
+ // -noautorotate keeps frames in their stored orientation
+ isFrameVertical(videoStream: VideoStreamInfo) {
+ return videoStream.height > videoStream.width;
+ }
+
getFilterOptions(videoStream: VideoStreamInfo) {
const options = [];
const tonemapOptions = this.getToneMapping(videoStream);
@@ -878,6 +889,11 @@ export class QsvHwDecodeConfig extends QsvSwDecodeConfig {
];
}
+ // -noautorotate keeps frames in their stored orientation
+ isFrameVertical(videoStream: VideoStreamInfo) {
+ return videoStream.height > videoStream.width;
+ }
+
getFilterOptions(videoStream: VideoStreamInfo) {
const options = [];
const tonemapOptions = this.getToneMapping(videoStream);
@@ -1000,6 +1016,11 @@ export class VaapiHwDecodeConfig extends VaapiSwDecodeConfig {
];
}
+ // -noautorotate keeps frames in their stored orientation
+ isFrameVertical(videoStream: VideoStreamInfo) {
+ return videoStream.height > videoStream.width;
+ }
+
getFilterOptions(videoStream: VideoStreamInfo) {
const options = [];
const tonemapOptions = this.getToneMapping(videoStream);
@@ -1087,6 +1108,11 @@ export class RkmppHwDecodeConfig extends RkmppSwDecodeConfig {
return ['-hwaccel', 'rkmpp', '-hwaccel_output_format', 'drm_prime', '-afbc', 'rga', '-noautorotate'];
}
+ // -noautorotate keeps frames in their stored orientation
+ isFrameVertical(videoStream: VideoStreamInfo) {
+ return videoStream.height > videoStream.width;
+ }
+
getFilterOptions(videoStream: VideoStreamInfo) {
if (this.shouldToneMap(videoStream)) {
const { primaries, transfer, matrix } = this.getColors();
diff --git a/server/test/fixtures/media.stub.ts b/server/test/fixtures/media.stub.ts
index 36939b5083..f0a52b285f 100644
--- a/server/test/fixtures/media.stub.ts
+++ b/server/test/fixtures/media.stub.ts
@@ -288,6 +288,31 @@ export const videoInfoStub = {
},
],
}),
+ videoStreamRotatedHorizontal2160p: Object.freeze<VideoInfo>({
+ ...probeStubDefault,
+ videoStreams: [
+ {
+ index: 0,
+ height: 3840,
+ width: 2160,
+ codecName: 'h264',
+ frameCount: 100,
+ rotation: 90,
+ bitrate: 0,
+ colorPrimaries: ColorPrimaries.Bt709,
+ colorTransfer: ColorTransfer.Bt709,
+ colorMatrix: ColorMatrix.Bt709,
+ pixelFormat: 'yuv420p',
+ frameRate: 60,
+ timeBase: 600,
+ profile: H264Profile.High,
+ level: null,
+ dvBlSignalCompatibilityId: null,
+ dvLevel: null,
+ dvProfile: null,
+ },
+ ],
+ }),
videoStreamOddHeight: Object.freeze<VideoInfo>({
...probeStubDefault,
videoStreams: [