Skip to content

High performance self-hosted photo and video management solution.

fix(server): av typing (#28223)

Mert01712cf0a702
diff --git a/server/src/repositories/media.repository.ts b/server/src/repositories/media.repository.ts
index ce81e7752b..088276aa17 100644
--- a/server/src/repositories/media.repository.ts
+++ b/server/src/repositories/media.repository.ts
@@ -274,23 +274,23 @@ export class MediaRepository {
index: stream.index,
height,
width: dar ? Math.round(height * dar) : this.parseInt(stream.width),
- codecName: stream.codec_name === 'h265' ? 'hevc' : stream.codec_name,
- profile: this.parseVideoProfile(stream.codec_name, stream.profile as string | undefined),
+ codecName: stream.codec_name === 'h265' ? 'hevc' : (stream.codec_name ?? null),
+ profile: this.parseVideoProfile(stream.codec_name, stream.profile as string | undefined) ?? null,
level: this.parseOptionalInt(stream.level),
frameCount: this.parseInt(options?.countFrames ? stream.nb_read_packets : stream.nb_frames),
frameRate: this.parseFrameRate(stream.avg_frame_rate ?? stream.r_frame_rate),
- timeBase: this.parseRational(stream.time_base)?.den,
+ timeBase: this.parseRational(stream.time_base)?.den ?? null,
rotation: this.parseInt(stream.rotation),
bitrate: this.parseInt(stream.bit_rate),
pixelFormat: stream.pix_fmt || 'yuv420p',
colorPrimaries: this.parseEnum(ColorPrimaries, stream.color_primaries) ?? ColorPrimaries.Unknown,
colorMatrix: this.parseEnum(ColorMatrix, stream.color_space) ?? ColorMatrix.Unknown,
colorTransfer: this.parseEnum(ColorTransfer, stream.color_transfer) ?? ColorTransfer.Unknown,
- dvProfile: this.parseOptionalInt(stream.dv_profile) as DvProfile | undefined,
+ dvProfile: this.parseOptionalInt(stream.dv_profile) as DvProfile | null,
dvLevel: this.parseOptionalInt(stream.dv_level),
- dvBlSignalCompatibilityId: this.parseOptionalInt(stream.dv_bl_signal_compatibility_id) as
- | DvSignalCompatibility
- | undefined,
+ dvBlSignalCompatibilityId: this.parseOptionalInt(
+ stream.dv_bl_signal_compatibility_id,
+ ) as DvSignalCompatibility | null,
};
}),
audioStreams: results.streams
@@ -298,9 +298,9 @@ export class MediaRepository {
.sort((a, b) => this.compareStreams(a, b))
.map((stream) => ({
index: stream.index,
- codecName: stream.codec_name,
+ codecName: stream.codec_name ?? null,
profile:
- stream.codec_name === 'aac' ? this.parseEnum(AacProfile, stream.profile as string | undefined) : undefined,
+ stream.codec_name === 'aac' ? this.parseEnum(AacProfile, stream.profile as string | undefined) : null,
bitrate: this.parseInt(stream.bit_rate),
})),
};
@@ -449,29 +449,29 @@ export class MediaRepository {
return Number.parseFloat(value as string) || 0;
}
- private parseOptionalInt(value: string | number | undefined): number | undefined {
+ private parseOptionalInt(value: string | number | undefined): number | null {
const parsed = Number.parseInt(value as string);
- return Number.isNaN(parsed) ? undefined : parsed;
+ return Number.isNaN(parsed) ? null : parsed;
}
private parseEnum<E extends Record<string, number | string>>(enumObj: E, value?: string) {
- return value ? (enumObj[pascalCase(value)] as Extract<E[keyof E], number> | undefined) : undefined;
+ return value ? ((enumObj[pascalCase(value)] as Extract<E[keyof E], number> | undefined) ?? null) : null;
}
/** Parse a rational like "60000/1001" or "1/600" into `{ num, den }`. */
- private parseRational(value: string | undefined): { num: number; den: number } | undefined {
- if (!value) {
- return;
- }
- const [num, den = 1] = value.split('/').map(Number);
- if (num && den) {
- return { num, den };
+ private parseRational(value: string | undefined): { num: number; den: number } | null {
+ if (value) {
+ const [num, den = 1] = value.split('/').map(Number);
+ if (num && den) {
+ return { num, den };
+ }
}
+ return null;
}
- private parseFrameRate(value: string | undefined): number | undefined {
+ private parseFrameRate(value: string | undefined): number | null {
const r = this.parseRational(value);
- return r ? r.num / r.den : undefined;
+ return r ? r.num / r.den : null;
}
private getDar(dar: string | undefined): number {
@@ -498,6 +498,7 @@ export class MediaRepository {
return this.parseEnum(Av1Profile, profile);
}
}
+ return null;
}
private compareStreams(a: FfprobeStream, b: FfprobeStream): number {
diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts
index cb29598c10..1524d96336 100644
--- a/server/src/services/media.service.spec.ts
+++ b/server/src/services/media.service.spec.ts
@@ -1,4 +1,4 @@
-import { NotNull, ShallowDehydrateObject } from 'kysely';
+import { ShallowDehydrateObject } from 'kysely';
import { OutputInfo } from 'sharp';
import { SystemConfig } from 'src/config';
import { Exif } from 'src/database';
@@ -1937,7 +1937,7 @@ describe(MediaService.name, () => {
describe('handleVideoConversion', () => {
let asset: ReturnType<typeof AssetFactory.create> & {
- videoStream: VideoStreamInfo & { timeBase: NotNull };
+ videoStream: VideoStreamInfo & { timeBase: number };
audioStream: AudioStreamInfo | null;
format: VideoFormat;
};
diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.ts
index 19efd50e99..f5ffe52375 100644
--- a/server/src/services/metadata.service.spec.ts
+++ b/server/src/services/metadata.service.spec.ts
@@ -672,7 +672,7 @@ describe(MetadataService.name, () => {
colorPrimaries: 9,
colorTransfer: 16,
colorMatrix: 9,
- dvProfile: undefined,
+ dvProfile: null,
}),
}),
);
diff --git a/server/src/types.ts b/server/src/types.ts
index 86ba0a1cc2..aa6bb820cc 100644
--- a/server/src/types.ts
+++ b/server/src/types.ts
@@ -89,26 +89,26 @@ export interface VideoStreamInfo {
height: number;
width: number;
rotation: number;
- codecName?: string;
- profile?: H264Profile | HevcProfile | Av1Profile;
- level?: number;
+ codecName: string | null;
+ profile: H264Profile | HevcProfile | Av1Profile | null;
+ level: number | null;
frameCount: number;
- frameRate?: number;
- timeBase?: number;
+ frameRate: number | null;
+ timeBase: number | null;
bitrate: number;
pixelFormat: string;
colorPrimaries: ColorPrimaries;
colorMatrix: ColorMatrix;
colorTransfer: ColorTransfer;
- dvProfile?: DvProfile;
- dvLevel?: number;
- dvBlSignalCompatibilityId?: DvSignalCompatibility;
+ dvProfile: DvProfile | null;
+ dvLevel: number | null;
+ dvBlSignalCompatibilityId: DvSignalCompatibility | null;
}
export interface AudioStreamInfo {
index: number;
- codecName?: string;
- profile?: AacProfile;
+ codecName: string | null;
+ profile: AacProfile | null;
bitrate: number;
}
diff --git a/server/src/utils/database.ts b/server/src/utils/database.ts
index fdf1d98caf..bc530f2b03 100644
--- a/server/src/utils/database.ts
+++ b/server/src/utils/database.ts
@@ -21,7 +21,7 @@ import { AssetFileType, AssetVisibility, DatabaseExtension, ExifOrientation } fr
import { AssetSearchBuilderOptions } from 'src/repositories/search.repository';
import { DB } from 'src/schema';
import { AssetExifTable } from 'src/schema/tables/asset-exif.table';
-import { AudioStreamInfo, VectorExtension, VideoFormat, VideoStreamInfo } from 'src/types';
+import { AudioStreamInfo, VectorExtension, VideoFormat, VideoPacketInfo, VideoStreamInfo } from 'src/types';
export const getKyselyConfig = (connection: DatabaseConnectionParams): KyselyConfig => {
return {
@@ -146,7 +146,7 @@ export function withVideoStream(eb: ExpressionBuilder<DB, 'asset_exif' | 'asset_
'asset_video.dvBlSignalCompatibilityId',
])
.where('asset_video.assetId', 'is not', sql.lit(null)),
- ).$castTo<(VideoStreamInfo & { timeBase: NotNull }) | null>();
+ ).$castTo<(VideoStreamInfo & { timeBase: number }) | null>();
}
export function withVideoFormat(eb: ExpressionBuilder<DB, 'asset' | 'asset_video'>) {
@@ -158,6 +158,22 @@ export function withVideoFormat(eb: ExpressionBuilder<DB, 'asset' | 'asset_video
).$castTo<VideoFormat | null>();
}
+export function withVideoPackets(eb: ExpressionBuilder<DB, 'asset' | 'asset_keyframe'>) {
+ return jsonObjectFrom(
+ eb
+ .selectFrom(dummy)
+ .where('asset_keyframe.assetId', 'is not', sql.lit(null))
+ .select([
+ 'asset_keyframe.pts as keyframePts',
+ 'asset_keyframe.accDuration as keyframeAccDuration',
+ 'asset_keyframe.ownDuration as keyframeOwnDuration',
+ 'asset_keyframe.totalDuration',
+ 'asset_keyframe.packetCount',
+ 'asset_keyframe.outputFrames',
+ ]),
+ ).$castTo<VideoPacketInfo | null>();
+}
+
export function withSmartSearch<O>(qb: SelectQueryBuilder<DB, 'asset', O>) {
return qb
.leftJoin('smart_search', 'asset.id', 'smart_search.assetId')
diff --git a/server/test/fixtures/media.stub.ts b/server/test/fixtures/media.stub.ts
index 120fe07664..f034ab873d 100644
--- a/server/test/fixtures/media.stub.ts
+++ b/server/test/fixtures/media.stub.ts
@@ -1,5 +1,13 @@
-import { NotNull } from 'kysely';
-import { ColorMatrix, ColorPrimaries, ColorTransfer, DvProfile, DvSignalCompatibility } from 'src/enum';
+import {
+ AacProfile,
+ ColorMatrix,
+ ColorPrimaries,
+ ColorTransfer,
+ DvProfile,
+ DvSignalCompatibility,
+ H264Profile,
+ HevcProfile,
+} from 'src/enum';
import { AudioStreamInfo, VideoFormat, VideoInfo, VideoStreamInfo } from 'src/types';
const probeStubDefaultFormat: VideoFormat = {
@@ -22,11 +30,17 @@ const probeStubDefaultVideoStream: VideoStreamInfo[] = [
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
+ frameRate: 60,
timeBase: 600,
+ profile: HevcProfile.Main,
+ level: null,
+ dvBlSignalCompatibilityId: null,
+ dvLevel: null,
+ dvProfile: null,
},
];
-const probeStubDefaultAudioStream: AudioStreamInfo[] = [{ index: 3, codecName: 'mp3', bitrate: 100 }];
+const probeStubDefaultAudioStream: AudioStreamInfo[] = [{ index: 3, codecName: 'mp3', bitrate: 100, profile: null }];
const probeStubDefault: VideoInfo = {
format: probeStubDefaultFormat,
@@ -53,7 +67,13 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
+ frameRate: 60,
timeBase: 600,
+ profile: HevcProfile.Main,
+ level: null,
+ dvBlSignalCompatibilityId: null,
+ dvLevel: null,
+ dvProfile: null,
},
{
index: 0,
@@ -67,7 +87,13 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
+ frameRate: 60,
timeBase: 600,
+ profile: HevcProfile.Main,
+ level: null,
+ dvBlSignalCompatibilityId: null,
+ dvLevel: null,
+ dvProfile: null,
},
{
index: 2,
@@ -81,16 +107,22 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
+ frameRate: 60,
timeBase: 600,
+ profile: HevcProfile.Main,
+ level: null,
+ dvBlSignalCompatibilityId: null,
+ dvLevel: null,
+ dvProfile: null,
},
],
}),
multipleAudioStreams: Object.freeze<VideoInfo>({
...probeStubDefault,
audioStreams: [
- { index: 2, codecName: 'mp3', bitrate: 102 },
- { index: 1, codecName: 'mp3', bitrate: 101 },
- { index: 0, codecName: 'mp3', bitrate: 100 },
+ { index: 2, codecName: 'mp3', bitrate: 102, profile: null },
+ { index: 1, codecName: 'mp3', bitrate: 101, profile: null },
+ { index: 0, codecName: 'mp3', bitrate: 100, profile: null },
],
}),
noHeight: Object.freeze<VideoInfo>({
@@ -108,7 +140,13 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
+ frameRate: 60,
timeBase: 600,
+ profile: HevcProfile.Main,
+ level: null,
+ dvBlSignalCompatibilityId: null,
+ dvLevel: null,
+ dvProfile: null,
},
],
}),
@@ -127,7 +165,13 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
+ frameRate: 60,
timeBase: 600,
+ profile: HevcProfile.Main,
+ level: null,
+ dvBlSignalCompatibilityId: null,
+ dvLevel: null,
+ dvProfile: null,
},
],
}),
@@ -159,7 +203,13 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Smpte2084,
bitrate: 0,
pixelFormat: 'yuv420p10le',
+ frameRate: 60,
timeBase: 600,
+ profile: H264Profile.High10,
+ level: null,
+ dvBlSignalCompatibilityId: null,
+ dvLevel: null,
+ dvProfile: null,
},
],
}),
@@ -178,7 +228,13 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p10le',
+ frameRate: 60,
timeBase: 600,
+ profile: H264Profile.High10,
+ level: null,
+ dvBlSignalCompatibilityId: null,
+ dvLevel: null,
+ dvProfile: null,
},
],
}),
@@ -197,7 +253,13 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p10le',
+ frameRate: 60,
timeBase: 600,
+ profile: H264Profile.High10,
+ level: null,
+ dvBlSignalCompatibilityId: null,
+ dvLevel: null,
+ dvProfile: null,
},
],
}),
@@ -216,7 +278,13 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
+ frameRate: 60,
timeBase: 600,
+ profile: H264Profile.High,
+ level: null,
+ dvBlSignalCompatibilityId: null,
+ dvLevel: null,
+ dvProfile: null,
},
],
}),
@@ -235,7 +303,13 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
+ frameRate: 60,
timeBase: 600,
+ profile: H264Profile.Main,
+ level: null,
+ dvBlSignalCompatibilityId: null,
+ dvLevel: null,
+ dvProfile: null,
},
],
}),
@@ -254,27 +328,33 @@ export const videoInfoStub = {
colorTransfer: ColorTransfer.Bt709,
colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
+ frameRate: 60,
timeBase: 600,
+ profile: H264Profile.Main,
+ level: null,
+ dvBlSignalCompatibilityId: null,
+ dvLevel: null,
+ dvProfile: null,
},
],
}),
audioStreamAac: Object.freeze<VideoInfo>({
...probeStubDefault,
- audioStreams: [{ index: 1, codecName: 'aac', bitrate: 100 }],
+ audioStreams: [{ index: 1, codecName: 'aac', bitrate: 100, profile: AacProfile.Lc }],
}),
audioStreamMp3: Object.freeze<VideoInfo>({
...probeStubDefault,
- audioStreams: [{ index: 1, codecName: 'mp3', bitrate: 100 }],
+ audioStreams: [{ index: 1, codecName: 'mp3', bitrate: 100, profile: null }],
}),
audioStreamOpus: Object.freeze<VideoInfo>({
...probeStubDefault,
- audioStreams: [{ index: 1, codecName: 'opus', bitrate: 100 }],
+ audioStreams: [{ index: 1, codecName: 'opus', bitrate: 100, profile: null }],
}),
audioStreamUnknown: Object.freeze<VideoInfo>({
...probeStubDefault,
audioStreams: [
- { index: 0, codecName: 'aac', bitrate: 100 },
- { index: 1, codecName: 'unknown', bitrate: 200 },
+ { index: 0, codecName: 'aac', bitrate: 100, profile: AacProfile.Lc },
+ { index: 1, codecName: 'unknown', bitrate: 200, profile: null },
],
}),
matroskaContainer: Object.freeze<VideoInfo>({
@@ -340,6 +420,9 @@ export const videoInfoStub = {
colorMatrix: ColorMatrix.Bt2020Nc,
colorTransfer: ColorTransfer.Smpte2084,
timeBase: 600,
+ dvBlSignalCompatibilityId: null,
+ dvLevel: null,
+ dvProfile: null,
},
],
}),
@@ -393,7 +476,7 @@ export const videoInfoStub = {
};
interface SelectedStreams {
- videoStream: VideoStreamInfo & { timeBase: NotNull };
+ videoStream: VideoStreamInfo & { timeBase: number };
audioStream: AudioStreamInfo | null;
format: VideoFormat;
}
@@ -407,3 +490,128 @@ const toSelectedStreams = (info: VideoInfo) => ({
export const probeStub = Object.fromEntries(
Object.entries(videoInfoStub).map(([key, info]) => [key, toSelectedStreams(info)]),
) as Record<keyof typeof videoInfoStub, SelectedStreams>;
+
+export const eiffelTower = {
+ originalPath: 'eiffel-tower.mp4',
+ videoStream: {
+ index: 0,
+ width: 1080,
+ height: 1920,
+ rotation: 0,
+ codecName: 'h264',
+ profile: H264Profile.High,
+ level: 40,
+ frameCount: 557,
+ frameRate: 24.908_004_845_459_07,
+ timeBase: 90_000,
+ bitrate: 5_128_622,
+ pixelFormat: 'yuv420p',
+ colorPrimaries: ColorPrimaries.Smpte170M,
+ colorTransfer: ColorTransfer.Smpte170M,
+ colorMatrix: ColorMatrix.Smpte170M,
+ dvProfile: null,
+ dvLevel: null,
+ dvBlSignalCompatibilityId: null,
+ },
+ audioStream: { codecName: 'aac', bitrate: 125_629, index: 1, profile: AacProfile.Lc },
+ packets: {
+ totalDuration: 2_012_441,
+ packetCount: 557,
+ outputFrames: 557,
+ keyframePts: [0, 462_502, 925_004, 1_210_454, 1_387_506, 1_542_878, 1_850_008],
+ keyframeAccDuration: [3613, 466_077, 928_541, 1_213_968, 1_391_005, 1_546_364, 1_853_469],
+ keyframeOwnDuration: [3613, 3613, 3613, 3613, 3613, 3613, 3613],
+ },
+ format: {
+ formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
+ formatLongName: 'QuickTime / MOV',
+ duration: 22_616,
+ bitrate: 5_128_622,
+ },
+};
+
+export const waterfall = {
+ originalPath: 'waterfall.mp4',
+ videoStream: {
+ index: 2,
+ width: 3840,
+ height: 2160,
+ rotation: -90,
+ codecName: 'hevc',
+ profile: HevcProfile.Main,
+ level: 156,
+ frameCount: 309,
+ frameRate: 29.829_901_982_867_92,
+ timeBase: 90_000,
+ bitrate: 43_363_499,
+ pixelFormat: 'yuvj420p',
+ colorPrimaries: ColorPrimaries.Bt709,
+ colorTransfer: ColorTransfer.Bt709,
+ colorMatrix: ColorMatrix.Bt709,
+ dvProfile: null,
+ dvLevel: null,
+ dvBlSignalCompatibilityId: null,
+ },
+ audioStream: { codecName: 'aac', bitrate: 191_878, index: 1, profile: null },
+ packets: {
+ totalDuration: 932_286,
+ packetCount: 309,
+ outputFrames: 309,
+ keyframePts: [0, 89_987, 179_974, 269_961, 359_948, 449_936, 539_923, 629_910, 725_166, 815_273, 905_295],
+ keyframeAccDuration: [
+ 2999, 92_987, 182_974, 272_961, 362_948, 452_934, 542_922, 632_909, 728_175, 818_274, 908_296,
+ ],
+ keyframeOwnDuration: [2999, 3000, 3000, 3000, 3000, 2998, 2999, 2999, 3009, 3001, 3001],
+ },
+ format: {
+ formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
+ formatLongName: 'QuickTime / MOV',
+ duration: 10_359,
+ bitrate: 43_363_499,
+ },
+};
+
+export const train = {
+ originalPath: 'train.mov',
+ videoStream: {
+ index: 0,
+ width: 1920,
+ height: 1080,
+ rotation: -90,
+ codecName: 'hevc',
+ profile: HevcProfile.Main10,
+ level: 123,
+ frameCount: 1229,
+ frameRate: 56.536_072_989_342_94,
+ timeBase: 600,
+ bitrate: 12_595_191,
+ pixelFormat: 'yuv420p10le',
+ colorPrimaries: ColorPrimaries.Bt2020,
+ colorTransfer: ColorTransfer.AribStdB67,
+ colorMatrix: ColorMatrix.Bt2020Nc,
+ dvProfile: DvProfile.Dvhe08,
+ dvLevel: 5,
+ dvBlSignalCompatibilityId: DvSignalCompatibility.Hlg,
+ },
+ audioStream: { codecName: 'aac', bitrate: 175_477, index: 1, profile: AacProfile.Lc },
+ packets: {
+ totalDuration: 12_290,
+ packetCount: 1229,
+ outputFrames: 1303,
+ keyframePts: [
+ 0, 601, 1201, 1802, 2402, 3003, 3604, 4204, 4805, 5405, 6006, 6607, 7207, 7808, 8408, 9009, 9609, 10_210, 10_811,
+ 11_411, 12_062, 12_703,
+ ],
+ keyframeAccDuration: [
+ 10, 580, 1180, 1780, 2380, 2980, 3580, 4180, 4780, 5380, 5980, 6580, 7180, 7780, 8380, 8980, 9580, 10_180, 10_780,
+ 11_380, 11_780, 12_100,
+ ],
+ keyframeOwnDuration: [10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10],
+ },
+ format: {
+ formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
+ formatLongName: 'QuickTime / MOV',
+ duration: 21_738,
+ bitrate: 12_595_191,
+ },
+};
diff --git a/server/test/mappers.ts b/server/test/mappers.ts
index 53d1d00a13..6ee7e52ac6 100644
--- a/server/test/mappers.ts
+++ b/server/test/mappers.ts
@@ -1,4 +1,4 @@
-import { NotNull, Selectable, ShallowDehydrateObject } from 'kysely';
+import { Selectable, ShallowDehydrateObject } from 'kysely';
import { MapAsset } from 'src/dtos/asset-response.dto';
import { AssetEditActionItem } from 'src/dtos/editing.dto';
import { ActivityTable } from 'src/schema/tables/activity.table';
@@ -156,7 +156,7 @@ export const getForGenerateThumbnail = (asset: ReturnType<AssetFactory['build']>
files: asset.files.map((file) => getDehydrated(file)),
exifInfo: getDehydrated(asset.exifInfo),
edits: asset.edits.map(({ action, parameters }) => ({ action, parameters })) as AssetEditActionItem[],
- videoStream: null as (VideoStreamInfo & { timeBase: NotNull }) | null,
+ videoStream: null as (VideoStreamInfo & { timeBase: number }) | null,
audioStream: null as AudioStreamInfo | null,
format: null as VideoFormat | null,
});
diff --git a/server/test/medium/specs/exif/audio-video.spec.ts b/server/test/medium/specs/exif/audio-video.spec.ts
index 430e7826f9..2f6af6594d 100644
--- a/server/test/medium/specs/exif/audio-video.spec.ts
+++ b/server/test/medium/specs/exif/audio-video.spec.ts
@@ -1,17 +1,9 @@
import { Kysely } from 'kysely';
import { resolve } from 'node:path';
-import {
- AacProfile,
- AssetType,
- ColorMatrix,
- ColorPrimaries,
- ColorTransfer,
- DvProfile,
- DvSignalCompatibility,
- H264Profile,
- HevcProfile,
-} from 'src/enum';
+import { AssetType } from 'src/enum';
import { DB } from 'src/schema';
+import { withAudioStream, withVideoFormat, withVideoPackets, withVideoStream } from 'src/utils/database';
+import { eiffelTower, train, waterfall } from 'test/fixtures/media.stub';
import { ExifTestContext, testAssetsDir } from 'test/medium.factory';
import { getKyselyDB } from 'test/utils';
@@ -21,122 +13,30 @@ beforeAll(async () => {
database = await getKyselyDB();
});
-const fixtures = [
- {
- file: 'eiffel-tower.mp4',
- video: {
- codecName: 'h264',
- formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
- formatLongName: 'QuickTime / MOV',
- pixelFormat: 'yuv420p',
- bitrate: 5_128_622,
- frameCount: 557,
- timeBase: 90_000,
- index: 0,
- profile: H264Profile.High,
- level: 40,
- colorPrimaries: ColorPrimaries.Smpte170M,
- colorTransfer: ColorTransfer.Smpte170M,
- colorMatrix: ColorMatrix.Smpte170M,
- dvProfile: null,
- dvLevel: null,
- dvBlSignalCompatibilityId: null,
- },
- audio: { codecName: 'aac', bitrate: 125_629, index: 1, profile: AacProfile.Lc },
- keyframes: {
- totalDuration: 2_012_441,
- packetCount: 557,
- outputFrames: 557,
- pts: [0, 462_502, 925_004, 1_210_454, 1_387_506, 1_542_878, 1_850_008],
- accDuration: [3613, 466_077, 928_541, 1_213_968, 1_391_005, 1_546_364, 1_853_469],
- ownDuration: [3613, 3613, 3613, 3613, 3613, 3613, 3613],
- },
- },
- {
- file: 'waterfall.mp4',
- video: {
- codecName: 'hevc',
- formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
- formatLongName: 'QuickTime / MOV',
- pixelFormat: 'yuvj420p',
- bitrate: 43_363_499,
- frameCount: 309,
- timeBase: 90_000,
- index: 2,
- profile: HevcProfile.Main,
- level: 156,
- colorPrimaries: ColorPrimaries.Bt709,
- colorTransfer: ColorTransfer.Bt709,
- colorMatrix: ColorMatrix.Bt709,
- dvProfile: null,
- dvLevel: null,
- dvBlSignalCompatibilityId: null,
- },
- audio: { codecName: 'aac', bitrate: 191_878, index: 1, profile: null },
- keyframes: {
- totalDuration: 932_286,
- packetCount: 309,
- outputFrames: 309,
- pts: [0, 89_987, 179_974, 269_961, 359_948, 449_936, 539_923, 629_910, 725_166, 815_273, 905_295],
- accDuration: [2999, 92_987, 182_974, 272_961, 362_948, 452_934, 542_922, 632_909, 728_175, 818_274, 908_296],
- ownDuration: [2999, 3000, 3000, 3000, 3000, 2998, 2999, 2999, 3009, 3001, 3001],
- },
- },
- {
- file: 'train.mov',
- video: {
- codecName: 'hevc',
- formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
- formatLongName: 'QuickTime / MOV',
- pixelFormat: 'yuv420p10le',
- bitrate: 12_595_191,
- frameCount: 1229,
- timeBase: 600,
- index: 0,
- profile: HevcProfile.Main10,
- level: 123,
- colorPrimaries: ColorPrimaries.Bt2020,
- colorTransfer: ColorTransfer.AribStdB67,
- colorMatrix: ColorMatrix.Bt2020Nc,
- dvProfile: DvProfile.Dvhe08,
- dvLevel: 5,
- dvBlSignalCompatibilityId: DvSignalCompatibility.Hlg,
- },
- audio: { codecName: 'aac', bitrate: 175_477, index: 1, profile: AacProfile.Lc },
- keyframes: {
- totalDuration: 12_290,
- packetCount: 1229,
- outputFrames: 1303,
- pts: [
- 0, 601, 1201, 1802, 2402, 3003, 3604, 4204, 4805, 5405, 6006, 6607, 7207, 7808, 8408, 9009, 9609, 10_210,
- 10_811, 11_411, 12_062, 12_703,
- ],
- accDuration: [
- 10, 580, 1180, 1780, 2380, 2980, 3580, 4180, 4780, 5380, 5980, 6580, 7180, 7780, 8380, 8980, 9580, 10_180,
- 10_780, 11_380, 11_780, 12_100,
- ],
- ownDuration: [10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10],
- },
- },
-];
-
-const isExpected = <T extends keyof DB>(name: T, id: string, expected: Omit<DB[T], 'assetId'>) => {
- const { table, ref } = database.dynamic;
- const res = database.selectFrom(table(name).as('t')).selectAll().where(ref('assetId'), '=', id).executeTakeFirst();
- return expect(res).resolves.toEqual({ ...expected, assetId: id });
-};
+const fixtures = [eiffelTower, waterfall, train];
describe('video metadata extraction', () => {
- it.each(fixtures)('$file', async ({ file, video, audio, keyframes }) => {
+ it.each(fixtures)('$originalPath', async ({ originalPath: path, videoStream, audioStream, packets, format }) => {
const ctx = new ExifTestContext(database);
const { user } = await ctx.newUser();
- const originalPath = resolve(testAssetsDir, 'videos', file);
+ const originalPath = resolve(testAssetsDir, 'videos', path);
const { asset } = await ctx.newAsset({ ownerId: user.id, originalPath, type: AssetType.Video });
await ctx.sut.handleMetadataExtraction({ id: asset.id });
- await isExpected('asset_audio', asset.id, audio);
- await isExpected('asset_video', asset.id, video);
- await isExpected('asset_keyframe', asset.id, keyframes);
+ const result = await database
+ .selectFrom('asset')
+ .innerJoin('asset_exif', 'asset.id', 'asset_exif.assetId')
+ .innerJoin('asset_video', 'asset.id', 'asset_video.assetId')
+ .innerJoin('asset_keyframe', 'asset.id', 'asset_keyframe.assetId')
+ .leftJoin('asset_audio', 'asset.id', 'asset_audio.assetId')
+ .where('asset.id', '=', asset.id)
+ .select((eb) => withVideoStream(eb).$notNull().as('videoStream'))
+ .select((eb) => withAudioStream(eb).as('audioStream'))
+ .select((eb) => withVideoPackets(eb).$notNull().as('packets'))
+ .select((eb) => withVideoFormat(eb).$notNull().as('format'))
+ .executeTakeFirst();
+
+ expect(result).toEqual({ videoStream, audioStream, packets, format });
});
});