Skip to content

High performance self-hosted photo and video management solution.

feat(server): track video metadata (#28023)

Mertf1d8ab8aae5c
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 4558b90866..46b1baed7e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -392,6 +392,8 @@ jobs:
node-version-file: './server/.nvmrc'
cache: 'pnpm'
cache-dependency-path: '**/pnpm-lock.yaml'
+ - name: Setup Mise
+ uses: immich-app/devtools/actions/use-mise@035e80a7d4355d5f087ffb95db9e4a0944c04e56 # use-mise-action-v1.1.3
- name: Run pnpm install
run: SHARP_IGNORE_GLOBAL_LIBVIPS=true pnpm install --frozen-lockfile
- name: Run medium tests
diff --git a/e2e/test-assets b/e2e/test-assets
index 0eac5a3738..6742055402 160000
--- a/e2e/test-assets
+++ b/e2e/test-assets
@@ -1 +1 @@
-Subproject commit 0eac5a37384c151be88381b41f9e28d8d59a4466
+Subproject commit 6742055402de1aa48f93d12ded7d18f4057f9d1f
diff --git a/mise.toml b/mise.toml
index b7398a0e8d..47862f7aca 100644
--- a/mise.toml
+++ b/mise.toml
@@ -26,6 +26,15 @@ version = "1.35.1"
bin = "dcm"
postinstall = "chmod +x $MISE_TOOL_INSTALL_PATH/dcm"
+[tools."github:jellyfin/jellyfin-ffmpeg"]
+version = "7.1.3-6"
+
+[tools."github:jellyfin/jellyfin-ffmpeg".platforms]
+linux-x64 = { asset_pattern = "jellyfin-ffmpeg_*_portable_linux64-gpl.tar.xz" }
+linux-arm64 = { asset_pattern = "jellyfin-ffmpeg_*_portable_linuxarm64-gpl.tar.xz" }
+macos-x64 = { asset_pattern = "jellyfin-ffmpeg_*_portable_mac64-gpl.tar.xz" }
+macos-arm64 = { asset_pattern = "jellyfin-ffmpeg_*_portable_macarm64-gpl.tar.xz" }
+
[settings]
experimental = true
pin = true
diff --git a/server/eslint.config.mjs b/server/eslint.config.mjs
index f80dbb4691..bdbac34da6 100644
--- a/server/eslint.config.mjs
+++ b/server/eslint.config.mjs
@@ -48,6 +48,7 @@ export default typescriptEslint.config([
'unicorn/import-style': 'off',
'unicorn/prefer-structured-clone': 'off',
'unicorn/no-for-loop': 'off',
+ 'unicorn/no-array-sort': 'off',
'@typescript-eslint/await-thenable': 'error',
'@typescript-eslint/no-misused-promises': 'error',
'require-await': 'off',
diff --git a/server/src/enum.ts b/server/src/enum.ts
index f7abb42c3a..74f1142fec 100644
--- a/server/src/enum.ts
+++ b/server/src/enum.ts
@@ -603,6 +603,133 @@ export enum ExifOrientation {
Rotate270CW = 8,
}
+/** ITU-T H.273 colour primaries codes. */
+export enum ColorPrimaries {
+ Reserved = 0,
+ Bt709 = 1,
+ Unknown = 2,
+ Bt470M = 4,
+ Bt470Bg = 5,
+ Smpte170M = 6,
+ Smpte240M = 7,
+ Film = 8,
+ Bt2020 = 9,
+ Smpte428 = 10,
+ Smpte431 = 11,
+ Smpte432 = 12,
+ Ebu3213 = 22,
+}
+
+/** ITU-T H.273 transfer characteristics codes. */
+export enum ColorTransfer {
+ Reserved = 0,
+ Bt709 = 1,
+ Unknown = 2,
+ Bt470M = 4,
+ Bt470Bg = 5,
+ Smpte170M = 6,
+ Smpte240M = 7,
+ Linear = 8,
+ Log100 = 9,
+ Log316 = 10,
+ Iec6196624 = 11,
+ Bt1361E = 12,
+ Iec6196621 = 13,
+ Bt202010 = 14,
+ Bt202012 = 15,
+ Smpte2084 = 16,
+ Smpte428 = 17,
+ AribStdB67 = 18,
+}
+
+/** ITU-T H.273 matrix coefficients codes. */
+export enum ColorMatrix {
+ Gbr = 0,
+ Bt709 = 1,
+ Unknown = 2,
+ Reserved = 3,
+ Fcc = 4,
+ Bt470Bg = 5,
+ Smpte170M = 6,
+ Smpte240M = 7,
+ Ycgco = 8,
+ Bt2020Nc = 9,
+ Bt2020C = 10,
+ Smpte2085 = 11,
+ ChromaDerivedNc = 12,
+ ChromaDerivedC = 13,
+ Ictcp = 14,
+}
+
+/** H.264 `profile_idc` values. */
+// H.264 has a few profiles that have the same value but different names, included so lookup by name works
+export enum H264Profile {
+ ConstrainedBaseline = 66,
+ // eslint-disable-next-line @typescript-eslint/no-duplicate-enum-values
+ Baseline = 66,
+ Main = 77,
+ Extended = 88,
+ ConstrainedHigh = 100,
+ // eslint-disable-next-line @typescript-eslint/no-duplicate-enum-values
+ ProgressiveHigh = 100,
+ // eslint-disable-next-line @typescript-eslint/no-duplicate-enum-values
+ High = 100,
+ High10 = 110,
+ High422 = 122,
+ High444Predictive = 244,
+}
+
+/** HEVC `profile_idc` values. */
+export enum HevcProfile {
+ Main = 1,
+ Main10 = 2,
+ MainStillPicture = 3,
+ Rext = 4,
+}
+
+/** AV1 `seq_profile` values. */
+export enum Av1Profile {
+ Main = 0,
+ High = 1,
+ Professional = 2,
+}
+
+/** MPEG-4 Audio Object Type values for AAC. */
+export enum AacProfile {
+ Main = 1,
+ Lc = 2,
+ Ssr = 3,
+ Ltp = 4,
+ HeAac = 5,
+ Ld = 23,
+ HeAacv2 = 29,
+ Eld = 39,
+ XheAac = 42,
+}
+
+/** Dolby Vision bitstream profile numbers from the DOVI configuration record. */
+export enum DvProfile {
+ Dvhe03 = 3,
+ Dvhe04 = 4,
+ Dvhe05 = 5,
+ Dvhe07 = 7,
+ Dvhe08 = 8,
+ Dvav09 = 9,
+ Dav110 = 10,
+}
+
+/**
+ * Dolby Vision base-layer signal-compatibility ID from the DOVI configuration record.
+ * Identifies what the base HEVC/AVC layer renders as on a non-DV decoder.
+ */
+export enum DvSignalCompatibility {
+ None = 0,
+ Hdr10 = 1,
+ Sdr709 = 2,
+ Hlg = 4,
+ Sdr2020 = 6,
+}
+
export enum DatabaseExtension {
Cube = 'cube',
EarthDistance = 'earthdistance',
diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sql
index 746ef6bfee..aa04603913 100644
--- a/server/src/queries/asset.job.repository.sql
+++ b/server/src/queries/asset.job.repository.sql
@@ -239,10 +239,68 @@ select
"asset_edit"."assetId" = "asset"."id"
) as agg
) as "edits",
- to_json("asset_exif") as "exifInfo"
+ to_json("asset_exif") as "exifInfo",
+ (
+ select
+ to_json(obj)
+ from
+ (
+ select
+ "asset_video"."index",
+ "asset_video"."codecName",
+ "asset_video"."profile",
+ "asset_video"."level",
+ "asset_video"."bitrate",
+ "asset_exif"."exifImageWidth" as "width",
+ "asset_exif"."exifImageHeight" as "height",
+ "asset_video"."pixelFormat",
+ "asset_video"."frameCount",
+ "asset_exif"."fps" as "frameRate",
+ "asset_video"."timeBase",
+ case
+ when "asset_exif"."orientation" = '6' then -90
+ when "asset_exif"."orientation" = '8' then 90
+ when "asset_exif"."orientation" = '3' then 180
+ else 0
+ end as "rotation",
+ "asset_video"."colorPrimaries",
+ "asset_video"."colorMatrix",
+ "asset_video"."colorTransfer",
+ "asset_video"."dvProfile",
+ "asset_video"."dvLevel",
+ "asset_video"."dvBlSignalCompatibilityId"
+ from
+ (
+ select
+ 1
+ ) as "dummy"
+ where
+ "asset_video"."assetId" is not null
+ ) as obj
+ ) as "videoStream",
+ (
+ select
+ to_json(obj)
+ from
+ (
+ select
+ "asset_video"."formatName",
+ "asset_video"."formatLongName",
+ "asset"."duration",
+ "asset_video"."bitrate"
+ from
+ (
+ select
+ 1
+ ) as "dummy"
+ where
+ "asset_video"."assetId" is not null
+ ) as obj
+ ) as "format"
from
"asset"
inner join "asset_exif" on "asset"."id" = "asset_exif"."assetId"
+ left join "asset_video" on "asset_video"."assetId" = "asset"."id"
where
"asset"."id" = $4
@@ -554,9 +612,88 @@ select
where
"asset_file"."assetId" = "asset"."id"
) as agg
- ) as "files"
+ ) as "files",
+ (
+ select
+ to_json(obj)
+ from
+ (
+ select
+ "asset_audio"."index",
+ "asset_audio"."codecName",
+ "asset_audio"."profile",
+ "asset_audio"."bitrate"
+ from
+ (
+ select
+ 1
+ ) as "dummy"
+ where
+ "asset_audio"."assetId" is not null
+ ) as obj
+ ) as "audioStream",
+ (
+ select
+ to_json(obj)
+ from
+ (
+ select
+ "asset_video"."index",
+ "asset_video"."codecName",
+ "asset_video"."profile",
+ "asset_video"."level",
+ "asset_video"."bitrate",
+ "asset_exif"."exifImageWidth" as "width",
+ "asset_exif"."exifImageHeight" as "height",
+ "asset_video"."pixelFormat",
+ "asset_video"."frameCount",
+ "asset_exif"."fps" as "frameRate",
+ "asset_video"."timeBase",
+ case
+ when "asset_exif"."orientation" = '6' then -90
+ when "asset_exif"."orientation" = '8' then 90
+ when "asset_exif"."orientation" = '3' then 180
+ else 0
+ end as "rotation",
+ "asset_video"."colorPrimaries",
+ "asset_video"."colorMatrix",
+ "asset_video"."colorTransfer",
+ "asset_video"."dvProfile",
+ "asset_video"."dvLevel",
+ "asset_video"."dvBlSignalCompatibilityId"
+ from
+ (
+ select
+ 1
+ ) as "dummy"
+ where
+ "asset_video"."assetId" is not null
+ ) as obj
+ ) as "videoStream",
+ (
+ select
+ to_json(obj)
+ from
+ (
+ select
+ "asset_video"."formatName",
+ "asset_video"."formatLongName",
+ "asset"."duration",
+ "asset_video"."bitrate"
+ from
+ (
+ select
+ 1
+ ) as "dummy"
+ where
+ "asset_video"."assetId" is not null
+ ) as obj
+ ) as "format"
from
"asset"
+ inner join "asset_exif" on "asset"."id" = "asset_exif"."assetId"
+ inner join "asset_video" on "asset_video"."assetId" = "asset"."id"
+ left join "asset_audio" on "asset_audio"."assetId" = "asset"."id"
where
"asset"."id" = $1
and "asset"."type" = 'VIDEO'
diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts
index 3765cad7ed..bab0c44a41 100644
--- a/server/src/repositories/asset-job.repository.ts
+++ b/server/src/repositories/asset-job.repository.ts
@@ -9,6 +9,7 @@ import { DB } from 'src/schema';
import {
anyUuid,
asUuid,
+ withAudioStream,
withDefaultVisibility,
withEdits,
withExif,
@@ -16,6 +17,8 @@ import {
withFaces,
withFilePath,
withFiles,
+ withVideoFormat,
+ withVideoStream,
} from 'src/utils/database';
import { mimeTypes } from 'src/utils/mime-types';
@@ -134,6 +137,9 @@ export class AssetJobRepository {
)
.select(withEdits)
.$call(withExifInner)
+ .leftJoin('asset_video', 'asset_video.assetId', 'asset.id')
+ .select((eb) => withVideoStream(eb).as('videoStream'))
+ .select((eb) => withVideoFormat(eb).as('format'))
.where('asset.id', '=', id)
.executeTakeFirst();
}
@@ -333,8 +339,14 @@ export class AssetJobRepository {
getForVideoConversion(id: string) {
return this.db
.selectFrom('asset')
+ .innerJoin('asset_exif', 'asset.id', 'asset_exif.assetId')
+ .innerJoin('asset_video', 'asset_video.assetId', 'asset.id')
+ .leftJoin('asset_audio', 'asset_audio.assetId', 'asset.id')
.select(['asset.id', 'asset.ownerId', 'asset.originalPath'])
.select(withFiles)
+ .select((eb) => withAudioStream(eb).as('audioStream'))
+ .select((eb) => withVideoStream(eb).$notNull().as('videoStream'))
+ .select((eb) => withVideoFormat(eb).$notNull().as('format'))
.where('asset.id', '=', id)
.where('asset.type', '=', sql.lit(AssetType.Video))
.executeTakeFirst();
diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts
index 784cf68b5b..0b706cacf9 100644
--- a/server/src/repositories/asset.repository.ts
+++ b/server/src/repositories/asset.repository.ts
@@ -19,6 +19,7 @@ import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';
import { AuthDto } from 'src/dtos/auth.dto';
import { AssetFileType, AssetOrder, AssetStatus, AssetType, AssetVisibility } from 'src/enum';
import { DB } from 'src/schema';
+import { AssetAudioTable, AssetKeyframeTable, AssetVideoTable } from 'src/schema/tables/asset-av.table';
import { AssetExifTable } from 'src/schema/tables/asset-exif.table';
import { AssetFileTable } from 'src/schema/tables/asset-file.table';
import { AssetJobStatusTable } from 'src/schema/tables/asset-job-status.table';
@@ -124,6 +125,14 @@ interface GetByIdsRelations {
edits?: boolean;
}
+type UpsertExifOptions = {
+ exif: Insertable<AssetExifTable>;
+ audio?: Insertable<AssetAudioTable>;
+ video?: Insertable<AssetVideoTable>;
+ keyframes?: Insertable<AssetKeyframeTable>;
+ lockedPropertiesBehavior: 'override' | 'append' | 'skip';
+};
+
const distinctLocked = <T extends LockableProperty[] | null>(eb: ExpressionBuilder<DB, 'asset_exif'>, columns: T) =>
sql<T>`nullif(array(select distinct unnest(${eb.ref('asset_exif.lockedProperties')} || ${columns})), '{}')`;
@@ -161,15 +170,76 @@ export class AssetRepository {
@GenerateSql({
params: [
- { dateTimeOriginal: DummyValue.DATE, lockedProperties: ['dateTimeOriginal'] },
- { lockedPropertiesBehavior: 'append' },
+ {
+ exif: { dateTimeOriginal: DummyValue.DATE, lockedProperties: ['dateTimeOriginal'] },
+ lockedPropertiesBehavior: 'append',
+ },
],
})
- async upsertExif(
- exif: Insertable<AssetExifTable>,
- { lockedPropertiesBehavior }: { lockedPropertiesBehavior: 'override' | 'append' | 'skip' },
- ): Promise<void> {
- await this.db
+ async upsertExif({ exif, audio, video, keyframes, lockedPropertiesBehavior }: UpsertExifOptions): Promise<void> {
+ let query = this.db;
+ if (audio) {
+ (query as any) = this.db.with('audio', (qb) =>
+ qb
+ .insertInto('asset_audio')
+ .values(audio)
+ .onConflict((oc) =>
+ oc.column('assetId').doUpdateSet(({ ref }) => ({
+ bitrate: ref('asset_audio.bitrate'),
+ index: ref('asset_audio.index'),
+ profile: ref('asset_audio.profile'),
+ codecName: ref('asset_audio.codecName'),
+ })),
+ ),
+ );
+ }
+
+ if (video) {
+ (query as any) = query.with('video', (qb) =>
+ qb
+ .insertInto('asset_video')
+ .values(video)
+ .onConflict((oc) =>
+ oc.column('assetId').doUpdateSet(({ ref }) => ({
+ bitrate: ref('asset_video.bitrate'),
+ timeBase: ref('asset_video.timeBase'),
+ index: ref('asset_video.index'),
+ profile: ref('asset_video.profile'),
+ level: ref('asset_video.level'),
+ colorPrimaries: ref('asset_video.colorPrimaries'),
+ colorTransfer: ref('asset_video.colorTransfer'),
+ colorMatrix: ref('asset_video.colorMatrix'),
+ dvProfile: ref('asset_video.dvProfile'),
+ dvLevel: ref('asset_video.dvLevel'),
+ dvBlSignalCompatibilityId: ref('asset_video.dvBlSignalCompatibilityId'),
+ codecName: ref('asset_video.codecName'),
+ formatName: ref('asset_video.formatName'),
+ formatLongName: ref('asset_video.formatLongName'),
+ pixelFormat: ref('asset_video.pixelFormat'),
+ })),
+ ),
+ );
+ }
+
+ if (keyframes) {
+ (query as any) = query.with('keyframe', (qb) =>
+ qb
+ .insertInto('asset_keyframe')
+ .values(keyframes)
+ .onConflict((oc) =>
+ oc.column('assetId').doUpdateSet(({ ref }) => ({
+ pts: ref('asset_keyframe.pts'),
+ accDuration: ref('asset_keyframe.accDuration'),
+ ownDuration: ref('asset_keyframe.ownDuration'),
+ totalDuration: ref('asset_keyframe.totalDuration'),
+ packetCount: ref('asset_keyframe.packetCount'),
+ outputFrames: ref('asset_keyframe.outputFrames'),
+ })),
+ ),
+ );
+ }
+
+ await query
.insertInto('asset_exif')
.values(exif)
.onConflict((oc) =>
diff --git a/server/src/repositories/media.repository.ts b/server/src/repositories/media.repository.ts
index 58e006171a..ce81e7752b 100644
--- a/server/src/repositories/media.repository.ts
+++ b/server/src/repositories/media.repository.ts
@@ -1,14 +1,30 @@
import { Injectable } from '@nestjs/common';
import { ExifDateTime, exiftool, WriteTags } from 'exiftool-vendored';
-import ffmpeg, { FfprobeData } from 'fluent-ffmpeg';
+import ffmpeg, { FfprobeData, FfprobeStream } from 'fluent-ffmpeg';
+import _ from 'lodash';
import { Duration } from 'luxon';
+import { execFile as execFileCb } from 'node:child_process';
import fs from 'node:fs/promises';
import { Writable } from 'node:stream';
+import { promisify } from 'node:util';
import sharp from 'sharp';
import { ORIENTATION_TO_SHARP_ROTATION } from 'src/constants';
import { Exif } from 'src/database';
import { AssetEditActionItem } from 'src/dtos/editing.dto';
-import { Colorspace, LogLevel, RawExtractedFormat } from 'src/enum';
+import {
+ AacProfile,
+ Av1Profile,
+ ColorMatrix,
+ ColorPrimaries,
+ Colorspace,
+ ColorTransfer,
+ DvProfile,
+ DvSignalCompatibility,
+ H264Profile,
+ HevcProfile,
+ LogLevel,
+ RawExtractedFormat,
+} from 'src/enum';
import { LoggingRepository } from 'src/repositories/logging.repository';
import {
DecodeToBufferOptions,
@@ -18,6 +34,7 @@ import {
ProbeOptions,
TranscodeCommand,
VideoInfo,
+ VideoPacketInfo,
} from 'src/types';
import { handlePromiseError } from 'src/utils/misc';
import { createAffineMatrix } from 'src/utils/transform';
@@ -26,9 +43,14 @@ const probe = (input: string, options: string[]): Promise<FfprobeData> =>
new Promise((resolve, reject) =>
ffmpeg.ffprobe(input, options, (error, data) => (error ? reject(error) : resolve(data))),
);
+
+const execFile = promisify(execFileCb);
+
sharp.concurrency(0);
sharp.cache({ files: 0 });
+const pascalCase = (str: string) => _.upperFirst(_.camelCase(str.toLowerCase()));
+
type ProgressEvent = {
frames: number;
currentFps: number;
@@ -244,6 +266,7 @@ export class MediaRepository {
},
videoStreams: results.streams
.filter((stream) => stream.codec_type === 'video' && !stream.disposition?.attached_pic)
+ .sort((a, b) => this.compareStreams(a, b))
.map((stream) => {
const height = this.parseInt(stream.height);
const dar = this.getDar(stream.display_aspect_ratio);
@@ -252,28 +275,98 @@ export class MediaRepository {
height,
width: dar ? Math.round(height * dar) : this.parseInt(stream.width),
codecName: stream.codec_name === 'h265' ? 'hevc' : stream.codec_name,
- codecType: stream.codec_type,
+ profile: this.parseVideoProfile(stream.codec_name, stream.profile as string | undefined),
+ level: this.parseOptionalInt(stream.level),
frameCount: this.parseInt(options?.countFrames ? stream.nb_read_packets : stream.nb_frames),
+ frameRate: this.parseFrameRate(stream.avg_frame_rate ?? stream.r_frame_rate),
+ timeBase: this.parseRational(stream.time_base)?.den,
rotation: this.parseInt(stream.rotation),
- isHDR: stream.color_transfer === 'smpte2084' || stream.color_transfer === 'arib-std-b67',
bitrate: this.parseInt(stream.bit_rate),
pixelFormat: stream.pix_fmt || 'yuv420p',
- colorPrimaries: stream.color_primaries,
- colorSpace: stream.color_space,
- colorTransfer: stream.color_transfer,
+ colorPrimaries: this.parseEnum(ColorPrimaries, stream.color_primaries) ?? ColorPrimaries.Unknown,
+ colorMatrix: this.parseEnum(ColorMatrix, stream.color_space) ?? ColorMatrix.Unknown,
+ colorTransfer: this.parseEnum(ColorTransfer, stream.color_transfer) ?? ColorTransfer.Unknown,
+ dvProfile: this.parseOptionalInt(stream.dv_profile) as DvProfile | undefined,
+ dvLevel: this.parseOptionalInt(stream.dv_level),
+ dvBlSignalCompatibilityId: this.parseOptionalInt(stream.dv_bl_signal_compatibility_id) as
+ | DvSignalCompatibility
+ | undefined,
};
}),
audioStreams: results.streams
.filter((stream) => stream.codec_type === 'audio')
+ .sort((a, b) => this.compareStreams(a, b))
.map((stream) => ({
index: stream.index,
- codecType: stream.codec_type,
codecName: stream.codec_name,
+ profile:
+ stream.codec_name === 'aac' ? this.parseEnum(AacProfile, stream.profile as string | undefined) : undefined,
bitrate: this.parseInt(stream.bit_rate),
})),
};
}
+ /**
+ * Needed for accurate segments, especially when remuxing, seeking and/or VFR is involved.
+ * Scanning packets for keyframes in JS is much faster than -skip_frame nokey since it avoids decoding the video.
+ */
+ async probePackets(input: string, streamIndex: number): Promise<VideoPacketInfo | null> {
+ const { stdout } = await execFile('ffprobe', [
+ '-v',
+ 'error',
+ '-select_streams',
+ String(streamIndex),
+ '-show_entries',
+ 'packet=pts,duration,flags',
+ '-of',
+ 'csv=p=0',
+ input,
+ ]);
+
+ let totalDuration = 0;
+ const keyframePts: number[] = [];
+ const keyframeAccDuration: number[] = [];
+ const keyframeOwnDuration: number[] = [];
+ const postDiscard: { pts: number; duration: number }[] = [];
+ for (const line of stdout.split('\n')) {
+ if (!line) {
+ continue;
+ }
+ const [ptsStr, durationStr, flags] = line.split(',');
+ const pts = Number.parseInt(ptsStr);
+ const duration = Number.parseInt(durationStr);
+ if (Number.isNaN(pts) || Number.isNaN(duration)) {
+ continue;
+ }
+ // Discarded packets don't contribute to packet count, but still contribute to video duration
+ totalDuration += duration;
+ if (flags[1] !== 'D') {
+ postDiscard.push({ pts, duration });
+ }
+ if (flags[0] === 'K') {
+ keyframePts.push(pts);
+ keyframeAccDuration.push(totalDuration);
+ // VFR content can have variable duration keyframes,
+ // so we need to track their duration separately for accurate segment boundaries.
+ // Non-keyframes are accounted for in totalDuration.
+ keyframeOwnDuration.push(duration);
+ }
+ }
+
+ if (postDiscard.length === 0) {
+ return null;
+ }
+
+ return {
+ totalDuration,
+ packetCount: postDiscard.length,
+ outputFrames: this.cfrOutputFrames(postDiscard, postDiscard.length / totalDuration),
+ keyframePts,
+ keyframeAccDuration,
+ keyframeOwnDuration,
+ };
+ }
+
transcode(input: string, output: string | Writable, options: TranscodeCommand): Promise<void> {
if (!options.twoPass) {
return new Promise((resolve, reject) => {
@@ -356,6 +449,31 @@ export class MediaRepository {
return Number.parseFloat(value as string) || 0;
}
+ private parseOptionalInt(value: string | number | undefined): number | undefined {
+ const parsed = Number.parseInt(value as string);
+ return Number.isNaN(parsed) ? undefined : parsed;
+ }
+
+ private parseEnum<E extends Record<string, number | string>>(enumObj: E, value?: string) {
+ return value ? (enumObj[pascalCase(value)] as Extract<E[keyof E], number> | undefined) : undefined;
+ }
+
+ /** Parse a rational like "60000/1001" or "1/600" into `{ num, den }`. */
+ private parseRational(value: string | undefined): { num: number; den: number } | undefined {
+ if (!value) {
+ return;
+ }
+ const [num, den = 1] = value.split('/').map(Number);
+ if (num && den) {
+ return { num, den };
+ }
+ }
+
+ private parseFrameRate(value: string | undefined): number | undefined {
+ const r = this.parseRational(value);
+ return r ? r.num / r.den : undefined;
+ }
+
private getDar(dar: string | undefined): number {
if (dar) {
const [darW, darH] = dar.split(':').map(Number);
@@ -366,4 +484,42 @@ export class MediaRepository {
return 0;
}
+
+ private parseVideoProfile(codec?: string, profile?: string) {
+ switch (codec) {
+ case 'h264': {
+ return this.parseEnum(H264Profile, profile);
+ }
+ case 'h265':
+ case 'hevc': {
+ return this.parseEnum(HevcProfile, profile);
+ }
+ case 'av1': {
+ return this.parseEnum(Av1Profile, profile);
+ }
+ }
+ }
+
+ private compareStreams(a: FfprobeStream, b: FfprobeStream): number {
+ const d = (b.disposition?.default ?? 0) - (a.disposition?.default ?? 0);
+ if (d !== 0) {
+ return d;
+ }
+ return this.parseInt(b.bit_rate) - this.parseInt(a.bit_rate);
+ }
+
+ private cfrOutputFrames(packets: { pts: number; duration: number }[], slotsPerTick: number) {
+ // Packets may be out of PTS order due to B-frames
+ packets.sort((a, b) => a.pts - b.pts);
+ const firstPts = packets[0].pts;
+ let outputFrames = 0;
+ let nextPts = 0;
+ for (const pkt of packets) {
+ const delta = (pkt.pts - firstPts) * slotsPerTick - nextPts + pkt.duration * slotsPerTick;
+ const nb = delta < -1.1 ? 0 : delta > 1.1 ? Math.round(delta) : 1;
+ outputFrames += nb;
+ nextPts += nb;
+ }
+ return outputFrames;
+ }
}
diff --git a/server/src/schema/index.ts b/server/src/schema/index.ts
index 469d58db1d..3bb7caf5ff 100644
--- a/server/src/schema/index.ts
+++ b/server/src/schema/index.ts
@@ -33,6 +33,7 @@ import { AlbumUserTable } from 'src/schema/tables/album-user.table';
import { AlbumTable } from 'src/schema/tables/album.table';
import { ApiKeyTable } from 'src/schema/tables/api-key.table';
import { AssetAuditTable } from 'src/schema/tables/asset-audit.table';
+import { AssetAudioTable, AssetKeyframeTable, AssetVideoTable } from 'src/schema/tables/asset-av.table';
import { AssetEditAuditTable } from 'src/schema/tables/asset-edit-audit.table';
import { AssetEditTable } from 'src/schema/tables/asset-edit.table';
import { AssetExifTable } from 'src/schema/tables/asset-exif.table';
@@ -204,6 +205,9 @@ export interface DB {
asset_metadata_audit: AssetMetadataAuditTable;
asset_job_status: AssetJobStatusTable;
asset_ocr: AssetOcrTable;
+ asset_audio: AssetAudioTable;
+ asset_video: AssetVideoTable;
+ asset_keyframe: AssetKeyframeTable;
ocr_search: OcrSearchTable;
face_search: FaceSearchTable;
diff --git a/server/src/schema/migrations/1777654048096-CreateAudioVideoTables.ts b/server/src/schema/migrations/1777654048096-CreateAudioVideoTables.ts
new file mode 100644
index 0000000000..2e0bf36bb0
--- /dev/null
+++ b/server/src/schema/migrations/1777654048096-CreateAudioVideoTables.ts
@@ -0,0 +1,51 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+ await sql`CREATE TABLE "asset_audio" (
+ "assetId" uuid NOT NULL,
+ "bitrate" integer NOT NULL,
+ "index" smallint NOT NULL,
+ "profile" smallint,
+ "codecName" text NOT NULL,
+ CONSTRAINT "asset_audio_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "asset" ("id") ON UPDATE NO ACTION ON DELETE CASCADE,
+ CONSTRAINT "asset_audio_pkey" PRIMARY KEY ("assetId")
+);`.execute(db);
+ await sql`CREATE TABLE "asset_video" (
+ "assetId" uuid NOT NULL,
+ "bitrate" integer NOT NULL,
+ "frameCount" integer NOT NULL,
+ "timeBase" integer NOT NULL,
+ "index" smallint NOT NULL,
+ "profile" smallint,
+ "level" smallint,
+ "colorPrimaries" smallint NOT NULL,
+ "colorTransfer" smallint NOT NULL,
+ "colorMatrix" smallint NOT NULL,
+ "dvProfile" smallint,
+ "dvLevel" smallint,
+ "dvBlSignalCompatibilityId" smallint,
+ "codecName" text NOT NULL,
+ "formatName" text NOT NULL,
+ "formatLongName" text NOT NULL,
+ "pixelFormat" text NOT NULL,
+ CONSTRAINT "asset_video_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "asset" ("id") ON UPDATE NO ACTION ON DELETE CASCADE,
+ CONSTRAINT "asset_video_pkey" PRIMARY KEY ("assetId")
+);`.execute(db);
+ await sql`CREATE TABLE "asset_keyframe" (
+ "assetId" uuid NOT NULL,
+ "pts" integer[] NOT NULL,
+ "accDuration" integer[] NOT NULL,
+ "ownDuration" integer[] NOT NULL,
+ "totalDuration" integer NOT NULL,
+ "packetCount" integer NOT NULL,
+ "outputFrames" integer NOT NULL,
+ CONSTRAINT "asset_keyframe_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "asset" ("id") ON UPDATE NO ACTION ON DELETE CASCADE,
+ CONSTRAINT "asset_keyframe_pkey" PRIMARY KEY ("assetId")
+);`.execute(db);
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+ await sql`DROP TABLE "asset_audio";`.execute(db);
+ await sql`DROP TABLE "asset_video";`.execute(db);
+ await sql`DROP TABLE "asset_keyframe";`.execute(db);
+}
diff --git a/server/src/schema/tables/asset-av.table.ts b/server/src/schema/tables/asset-av.table.ts
new file mode 100644
index 0000000000..41824e2509
--- /dev/null
+++ b/server/src/schema/tables/asset-av.table.ts
@@ -0,0 +1,98 @@
+import { Column, ForeignKeyColumn, Table } from '@immich/sql-tools';
+import { AssetTable } from 'src/schema/tables/asset.table';
+
+@Table('asset_audio')
+export class AssetAudioTable {
+ @ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', primary: true })
+ assetId!: string;
+
+ @Column({ type: 'integer' })
+ bitrate!: number;
+
+ @Column({ type: 'smallint' })
+ index!: number;
+
+ @Column({ type: 'smallint', nullable: true })
+ profile!: number | null;
+
+ @Column({ type: 'text' })
+ codecName!: string;
+}
+
+@Table('asset_video')
+export class AssetVideoTable {
+ @ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', primary: true })
+ assetId!: string;
+
+ @Column({ type: 'integer' })
+ bitrate!: number;
+
+ @Column({ type: 'integer' })
+ frameCount!: number;
+
+ @Column({ type: 'integer' })
+ timeBase!: number;
+
+ @Column({ type: 'smallint' })
+ index!: number;
+
+ @Column({ type: 'smallint', nullable: true })
+ profile!: number | null;
+
+ @Column({ type: 'smallint', nullable: true })
+ level!: number | null;
+
+ @Column({ type: 'smallint' })
+ colorPrimaries!: number;
+
+ @Column({ type: 'smallint' })
+ colorTransfer!: number;
+
+ @Column({ type: 'smallint' })
+ colorMatrix!: number;
+
+ @Column({ type: 'smallint', nullable: true })
+ dvProfile!: number | null;
+
+ @Column({ type: 'smallint', nullable: true })
+ dvLevel!: number | null;
+
+ @Column({ type: 'smallint', nullable: true })
+ dvBlSignalCompatibilityId!: number | null;
+
+ @Column({ type: 'text' })
+ codecName!: string;
+
+ @Column({ type: 'text' })
+ formatName!: string;
+
+ @Column({ type: 'text' })
+ formatLongName!: string;
+
+ @Column({ type: 'text' })
+ pixelFormat!: string;
+}
+
+@Table('asset_keyframe')
+export class AssetKeyframeTable {
+ @ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', primary: true })
+ assetId!: string;
+
+ @Column({ type: 'integer', array: true })
+ pts!: number[];
+
+ @Column({ type: 'integer', array: true })
+ accDuration!: number[];
+
+ @Column({ type: 'integer', array: true })
+ ownDuration!: number[];
+
+ @Column({ type: 'integer' })
+ totalDuration!: number;
+
+ @Column({ type: 'integer' })
+ packetCount!: number;
+
+ @Column({ type: 'integer' })
+ outputFrames!: number;
+}
diff --git a/server/src/services/asset-media.service.ts b/server/src/services/asset-media.service.ts
index 74aaa8fcbd..6b0d73b77b 100644
--- a/server/src/services/asset-media.service.ts
+++ b/server/src/services/asset-media.service.ts
@@ -351,10 +351,10 @@ export class AssetMediaService extends BaseService {
await this.storageRepository.utimes(sidecarFile.originalPath, new Date(), new Date(dto.fileModifiedAt));
}
await this.storageRepository.utimes(file.originalPath, new Date(), new Date(dto.fileModifiedAt));
- await this.assetRepository.upsertExif(
- { assetId: asset.id, fileSizeInByte: file.size },
- { lockedPropertiesBehavior: 'override' },
- );
+ await this.assetRepository.upsertExif({
+ exif: { assetId: asset.id, fileSizeInByte: file.size },
+ lockedPropertiesBehavior: 'override',
+ });
await this.eventRepository.emit('AssetCreate', { asset });
diff --git a/server/src/services/asset.service.spec.ts b/server/src/services/asset.service.spec.ts
index 13462a3246..75e7fc5e87 100755
--- a/server/src/services/asset.service.spec.ts
+++ b/server/src/services/asset.service.spec.ts
@@ -187,8 +187,10 @@ describe(AssetService.name, () => {
await sut.update(authStub.admin, asset.id, { description: 'Test description' });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
- { assetId: asset.id, description: 'Test description', lockedProperties: ['description'] },
- { lockedPropertiesBehavior: 'append' },
+ expect.objectContaining({
+ exif: { assetId: asset.id, description: 'Test description', lockedProperties: ['description'] },
+ lockedPropertiesBehavior: 'append',
+ }),
);
});
@@ -201,12 +203,14 @@ describe(AssetService.name, () => {
await sut.update(authStub.admin, asset.id, { rating: 3 });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
- {
- assetId: asset.id,
- rating: 3,
- lockedProperties: ['rating'],
- },
- { lockedPropertiesBehavior: 'append' },
+ expect.objectContaining({
+ exif: {
+ assetId: asset.id,
+ rating: 3,
+ lockedProperties: ['rating'],
+ },
+ lockedPropertiesBehavior: 'append',
+ }),
);
});
diff --git a/server/src/services/asset.service.ts b/server/src/services/asset.service.ts
index 613029fe3c..e2d2d95f81 100644
--- a/server/src/services/asset.service.ts
+++ b/server/src/services/asset.service.ts
@@ -517,13 +517,13 @@ export class AssetService extends BaseService {
);
if (Object.keys(writes).length > 0) {
- await this.assetRepository.upsertExif(
- updateLockedColumns({
+ await this.assetRepository.upsertExif({
+ exif: updateLockedColumns({
assetId: id,
...writes,
}),
- { lockedPropertiesBehavior: 'append' },
- );
+ lockedPropertiesBehavior: 'append',
+ });
await this.jobRepository.queue({ name: JobName.SidecarWrite, data: { id } });
}
}
diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts
index 694cd8ccf9..cb29598c10 100644
--- a/server/src/services/media.service.spec.ts
+++ b/server/src/services/media.service.spec.ts
@@ -1,4 +1,4 @@
-import { ShallowDehydrateObject } from 'kysely';
+import { NotNull, ShallowDehydrateObject } from 'kysely';
import { OutputInfo } from 'sharp';
import { SystemConfig } from 'src/config';
import { Exif } from 'src/database';
@@ -21,7 +21,7 @@ import {
VideoCodec,
} from 'src/enum';
import { MediaService } from 'src/services/media.service';
-import { JobCounts, RawImageInfo } from 'src/types';
+import { AudioStreamInfo, JobCounts, RawImageInfo, VideoFormat, VideoStreamInfo } from 'src/types';
import { AssetFaceFactory } from 'test/factories/asset-face.factory';
import { AssetFactory } from 'test/factories/asset.factory';
import { PersonFactory } from 'test/factories/person.factory';
@@ -375,15 +375,16 @@ describe(MediaService.name, () => {
mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(getForGenerateThumbnail(asset));
await expect(sut.handleGenerateThumbnails({ id: asset.id })).resolves.toBe(JobStatus.Skipped);
- expect(mocks.media.probe).not.toHaveBeenCalled();
expect(mocks.media.generateThumbnail).not.toHaveBeenCalled();
expect(mocks.asset.update).not.toHaveBeenCalledWith();
});
it('should skip video thumbnail generation if no video stream', async () => {
const asset = AssetFactory.from({ type: AssetType.Video }).exif().build();
- mocks.media.probe.mockResolvedValue(probeStub.noVideoStreams);
- mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(getForGenerateThumbnail(asset));
+ mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue({
+ ...getForGenerateThumbnail(asset),
+ ...probeStub.noVideoStreams,
+ });
await expect(sut.handleGenerateThumbnails({ id: asset.id })).rejects.toThrowError();
expect(mocks.media.generateThumbnail).not.toHaveBeenCalled();
expect(mocks.asset.update).not.toHaveBeenCalledWith();
@@ -495,8 +496,10 @@ describe(MediaService.name, () => {
it('should generate a thumbnail for a video', async () => {
const asset = AssetFactory.from({ type: AssetType.Video, originalPath: '/original/path.ext' }).exif().build();
- mocks.media.probe.mockResolvedValue(probeStub.videoStream2160p);
- mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(getForGenerateThumbnail(asset));
+ mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue({
+ ...getForGenerateThumbnail(asset),
+ ...probeStub.videoStream2160p,
+ });
await sut.handleGenerateThumbnails({ id: asset.id });
expect(mocks.storage.mkdirSync).toHaveBeenCalledWith(expect.any(String));
@@ -505,7 +508,7 @@ describe(MediaService.name, () => {
expect.any(String),
expect.objectContaining({
inputOptions: ['-skip_frame', 'nointra', '-sws_flags', 'accurate_rnd+full_chroma_int'],
- outputOptions: [
+ outputOptions: expect.arrayContaining([
'-fps_mode',
'vfr',
'-frames:v',
@@ -516,7 +519,7 @@ describe(MediaService.name, () => {
'verbose',
'-vf',
String.raw`fps=12:start_time=0:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,scale=-2:1440:flags=lanczos+accurate_rnd+full_chroma_int:out_range=pc`,
- ],
+ ]),
twoPass: false,
}),
);
@@ -542,8 +545,10 @@ describe(MediaService.name, () => {
it('should tonemap thumbnail for hdr video', async () => {
const asset = AssetFactory.from({ type: AssetType.Video, originalPath: '/original/path.ext' }).exif().build();
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamHDR);
- mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(getForGenerateThumbnail(asset));
+ mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue({
+ ...getForGenerateThumbnail(asset),
+ ...probeStub.videoStreamHDR,
+ });
await sut.handleGenerateThumbnails({ id: asset.id });
expect(mocks.storage.mkdirSync).toHaveBeenCalledWith(expect.any(String));
@@ -552,7 +557,7 @@ describe(MediaService.name, () => {
expect.any(String),
expect.objectContaining({
inputOptions: ['-skip_frame', 'nointra', '-sws_flags', 'accurate_rnd+full_chroma_int'],
- outputOptions: [
+ outputOptions: expect.arrayContaining([
'-fps_mode',
'vfr',
'-frames:v',
@@ -562,8 +567,8 @@ describe(MediaService.name, () => {
'-v',
'verbose',
'-vf',
- String.raw`fps=12:start_time=0:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p`,
- ],
+ String.raw`fps=12:start_time=0:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,scale=-2:250:flags=lanczos+accurate_rnd+full_chroma_int:out_range=pc,tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p`,
+ ]),
twoPass: false,
}),
);
@@ -589,11 +594,13 @@ describe(MediaService.name, () => {
it('should always generate video thumbnail in one pass', async () => {
const asset = AssetFactory.from({ type: AssetType.Video, originalPath: '/original/path.ext' }).exif().build();
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamHDR);
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { twoPass: true, maxBitrate: '5000k' },
});
- mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(getForGenerateThumbnail(asset));
+ mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue({
+ ...getForGenerateThumbnail(asset),
+ ...probeStub.videoStreamHDR,
+ });
await sut.handleGenerateThumbnails({ id: asset.id });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -601,7 +608,7 @@ describe(MediaService.name, () => {
expect.any(String),
expect.objectContaining({
inputOptions: ['-skip_frame', 'nointra', '-sws_flags', 'accurate_rnd+full_chroma_int'],
- outputOptions: [
+ outputOptions: expect.arrayContaining([
'-fps_mode',
'vfr',
'-frames:v',
@@ -611,8 +618,8 @@ describe(MediaService.name, () => {
'-v',
'verbose',
'-vf',
- String.raw`fps=12:start_time=0:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p`,
- ],
+ String.raw`fps=12:start_time=0:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,scale=-2:250:flags=lanczos+accurate_rnd+full_chroma_int:out_range=pc,tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p`,
+ ]),
twoPass: false,
}),
);
@@ -620,8 +627,10 @@ describe(MediaService.name, () => {
it('should not skip intra frames for MTS file', async () => {
const asset = AssetFactory.from({ type: AssetType.Video, originalPath: '/original/path.ext' }).exif().build();
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamMTS);
- mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(getForGenerateThumbnail(asset));
+ mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue({
+ ...getForGenerateThumbnail(asset),
+ ...probeStub.videoStreamMTS,
+ });
await sut.handleGenerateThumbnails({ id: asset.id });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -638,8 +647,10 @@ describe(MediaService.name, () => {
it('should override reserved color metadata', async () => {
const asset = AssetFactory.from({ type: AssetType.Video, originalPath: '/original/path.ext' }).exif().build();
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamReserved);
- mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(getForGenerateThumbnail(asset));
+ mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue({
+ ...getForGenerateThumbnail(asset),
+ ...probeStub.videoStreamReserved,
+ });
await sut.handleGenerateThumbnails({ id: asset.id });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -659,9 +670,11 @@ describe(MediaService.name, () => {
it('should use scaling divisible by 2 even when using quick sync', async () => {
const asset = AssetFactory.from({ type: AssetType.Video, originalPath: '/original/path.ext' }).exif().build();
- mocks.media.probe.mockResolvedValue(probeStub.videoStream2160p);
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { accel: TranscodeHardwareAcceleration.Qsv } });
- mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(getForGenerateThumbnail(asset));
+ mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue({
+ ...getForGenerateThumbnail(asset),
+ ...probeStub.videoStream2160p,
+ });
await sut.handleGenerateThumbnails({ id: asset.id });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -855,11 +868,13 @@ describe(MediaService.name, () => {
it('should never set isProgressive for videos', async () => {
const asset = AssetFactory.from({ type: AssetType.Video, originalPath: '/original/path.ext' }).exif().build();
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamHDR);
mocks.systemMetadata.get.mockResolvedValue({
image: { preview: { progressive: true }, thumbnail: { progressive: true } },
});
- mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(getForGenerateThumbnail(asset));
+ mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue({
+ ...getForGenerateThumbnail(asset),
+ ...probeStub.videoStreamHDR,
+ });
await sut.handleGenerateThumbnails({ id: asset.id });
@@ -1921,26 +1936,33 @@ describe(MediaService.name, () => {
});
describe('handleVideoConversion', () => {
+ let asset: ReturnType<typeof AssetFactory.create> & {
+ videoStream: VideoStreamInfo & { timeBase: NotNull };
+ audioStream: AudioStreamInfo | null;
+ format: VideoFormat;
+ };
beforeEach(() => {
- const asset = AssetFactory.create({ id: 'video-id', type: AssetType.Video, originalPath: '/original/path.ext' });
+ asset = {
+ ...AssetFactory.create({ id: 'video-id', type: AssetType.Video, originalPath: '/original/path.ext' }),
+ videoStream: probeStub.videoStreamH264.videoStream,
+ audioStream: null,
+ format: probeStub.videoStreamH264.format,
+ };
mocks.assetJob.getForVideoConversion.mockResolvedValue(asset);
sut.videoInterfaces = { dri: ['renderD128'], mali: true };
});
it('should skip transcoding if asset not found', async () => {
- mocks.assetJob.getForVideoConversion.mockResolvedValue(void 0);
await sut.handleVideoConversion({ id: 'video-id' });
- expect(mocks.media.probe).not.toHaveBeenCalled();
expect(mocks.media.transcode).not.toHaveBeenCalled();
});
it('should transcode the highest bitrate video stream', async () => {
mocks.logger.isLevelEnabled.mockReturnValue(false);
- mocks.media.probe.mockResolvedValue(probeStub.multipleVideoStreams);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.multipleVideoStreams });
await sut.handleVideoConversion({ id: 'video-id' });
- expect(mocks.media.probe).toHaveBeenCalledWith('/original/path.ext', { countFrames: false });
expect(mocks.systemMetadata.get).toHaveBeenCalled();
expect(mocks.storage.mkdirSync).toHaveBeenCalled();
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -1956,11 +1978,10 @@ describe(MediaService.name, () => {
it('should transcode the highest bitrate audio stream', async () => {
mocks.logger.isLevelEnabled.mockReturnValue(false);
- mocks.media.probe.mockResolvedValue(probeStub.multipleAudioStreams);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.multipleAudioStreams });
await sut.handleVideoConversion({ id: 'video-id' });
- expect(mocks.media.probe).toHaveBeenCalledWith('/original/path.ext', { countFrames: false });
expect(mocks.systemMetadata.get).toHaveBeenCalled();
expect(mocks.storage.mkdirSync).toHaveBeenCalled();
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -1975,19 +1996,19 @@ describe(MediaService.name, () => {
});
it('should skip a video without any streams', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.noVideoStreams);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.noVideoStreams });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).not.toHaveBeenCalled();
});
it('should skip a video without any height', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.noHeight);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.noHeight });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).not.toHaveBeenCalled();
});
it('should throw an error if an unknown transcode policy is configured', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.noAudioStreams);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.noAudioStreams });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: 'foo' } } as never as SystemConfig);
await expect(sut.handleVideoConversion({ id: 'video-id' })).rejects.toThrowError();
@@ -1995,7 +2016,7 @@ describe(MediaService.name, () => {
});
it('should throw an error if transcoding fails and hw acceleration is disabled', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.multipleVideoStreams);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.multipleVideoStreams });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { transcode: TranscodePolicy.All, accel: TranscodeHardwareAcceleration.Disabled },
});
@@ -2006,7 +2027,7 @@ describe(MediaService.name, () => {
});
it('should transcode when set to all', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.multipleVideoStreams);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.multipleVideoStreams });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.All } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2021,7 +2042,7 @@ describe(MediaService.name, () => {
});
it('should transcode when optimal and too big', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStream2160p);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStream2160p });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.Optimal } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2036,14 +2057,14 @@ describe(MediaService.name, () => {
});
it('should not transcode when policy bitrate and bitrate lower than max bitrate', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStream40Mbps);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStream40Mbps });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.Bitrate, maxBitrate: '50M' } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).not.toHaveBeenCalled();
});
it('should transcode when policy bitrate and bitrate higher than max bitrate', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStream40Mbps);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStream40Mbps });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.Bitrate, maxBitrate: '30M' } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2058,21 +2079,21 @@ describe(MediaService.name, () => {
});
it('should not transcode when max bitrate is not a number', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStream40Mbps);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStream40Mbps });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.Bitrate, maxBitrate: 'foo' } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).not.toHaveBeenCalled();
});
it('should not transcode when max bitrate is 0', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStream40Mbps);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStream40Mbps });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.Bitrate, maxBitrate: '0' } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).not.toHaveBeenCalled();
});
it('should not scale resolution if no target resolution', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStream2160p);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStream2160p });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { transcode: TranscodePolicy.All, targetResolution: 'original' },
});
@@ -2089,7 +2110,7 @@ describe(MediaService.name, () => {
});
it('should scale horizontally when video is horizontal', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStream2160p);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStream2160p });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.Optimal } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2104,7 +2125,7 @@ describe(MediaService.name, () => {
});
it('should scale vertically when video is vertical', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamVertical2160p);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamVertical2160p });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.Optimal } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2119,7 +2140,7 @@ describe(MediaService.name, () => {
});
it('should always scale video if height is uneven', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamOddHeight);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamOddHeight });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { transcode: TranscodePolicy.All, targetResolution: 'original' },
});
@@ -2136,7 +2157,7 @@ describe(MediaService.name, () => {
});
it('should always scale video if width is uneven', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamOddWidth);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamOddWidth });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { transcode: TranscodePolicy.All, targetResolution: 'original' },
});
@@ -2153,7 +2174,7 @@ describe(MediaService.name, () => {
});
it('should copy video stream when video matches target', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { targetVideoCodec: VideoCodec.Hevc, acceptedAudioCodecs: [AudioCodec.Aac] },
});
@@ -2170,7 +2191,7 @@ describe(MediaService.name, () => {
});
it('should not include hevc tag when target is hevc and video stream is copied from a different codec', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamH264);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamH264 });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: {
targetVideoCodec: VideoCodec.Hevc,
@@ -2191,7 +2212,7 @@ describe(MediaService.name, () => {
});
it('should include hevc tag when target is hevc and copying hevc video stream', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: {
targetVideoCodec: VideoCodec.Hevc,
@@ -2212,7 +2233,7 @@ describe(MediaService.name, () => {
});
it('should copy audio stream when audio matches target', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.audioStreamAac);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.audioStreamAac });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.Optimal } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2227,7 +2248,7 @@ describe(MediaService.name, () => {
});
it('should remux when input is not an accepted container', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamAvi);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamAvi });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
'/original/path.ext',
@@ -2241,7 +2262,7 @@ describe(MediaService.name, () => {
});
it('should throw an exception if transcode value is invalid', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStream2160p);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStream2160p });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: 'invalid' as any } });
await expect(sut.handleVideoConversion({ id: 'video-id' })).rejects.toThrowError();
@@ -2249,35 +2270,34 @@ describe(MediaService.name, () => {
});
it('should not transcode if transcoding is disabled', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStream2160p);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStream2160p });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.Disabled } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).not.toHaveBeenCalled();
});
it('should not remux when input is not an accepted container and transcoding is disabled', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.Disabled } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).not.toHaveBeenCalled();
});
it('should not transcode if target codec is invalid', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStream2160p);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStream2160p });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { targetVideoCodec: 'invalid' as any } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).not.toHaveBeenCalled();
});
it('should delete existing transcode if current policy does not require transcoding', async () => {
- const asset = AssetFactory.from({ type: AssetType.Video })
+ const localAsset = AssetFactory.from({ type: AssetType.Video })
.file({ type: AssetFileType.EncodedVideo, path: '/encoded/video/path.mp4' })
.build();
- mocks.media.probe.mockResolvedValue(probeStub.videoStream2160p);
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.Disabled } });
- mocks.assetJob.getForVideoConversion.mockResolvedValue(asset);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...localAsset, ...probeStub.videoStream2160p });
- await sut.handleVideoConversion({ id: asset.id });
+ await sut.handleVideoConversion({ id: localAsset.id });
expect(mocks.media.transcode).not.toHaveBeenCalled();
expect(mocks.job.queue).toHaveBeenCalledWith({
@@ -2287,7 +2307,7 @@ describe(MediaService.name, () => {
});
it('should set max bitrate if above 0', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { maxBitrate: '4500k' } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2302,7 +2322,7 @@ describe(MediaService.name, () => {
});
it('should default max bitrate to kbps if no unit is provided', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { maxBitrate: '4500' } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2317,7 +2337,7 @@ describe(MediaService.name, () => {
});
it('should transcode in two passes for h264/h265 when enabled and max bitrate is above 0', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { twoPass: true, maxBitrate: '4500k' } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2341,7 +2361,7 @@ describe(MediaService.name, () => {
});
it('should fallback to one pass for h264/h265 if two-pass is enabled but no max bitrate is set', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { twoPass: true } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2356,7 +2376,7 @@ describe(MediaService.name, () => {
});
it('should transcode by bitrate in two passes for vp9 when two pass mode and max bitrate are enabled', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: {
maxBitrate: '4500k',
@@ -2377,7 +2397,7 @@ describe(MediaService.name, () => {
});
it('should transcode by crf in two passes for vp9 when two pass mode is enabled and max bitrate is disabled', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: {
maxBitrate: '0',
@@ -2398,7 +2418,7 @@ describe(MediaService.name, () => {
});
it('should configure preset for vp9', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { targetVideoCodec: VideoCodec.Vp9, preset: 'slow' } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2413,7 +2433,7 @@ describe(MediaService.name, () => {
});
it('should not configure preset for vp9 if invalid', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { preset: 'invalid', targetVideoCodec: VideoCodec.Vp9 } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2428,7 +2448,7 @@ describe(MediaService.name, () => {
});
it('should configure threads if above 0', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { targetVideoCodec: VideoCodec.Vp9, threads: 2 } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2443,7 +2463,7 @@ describe(MediaService.name, () => {
});
it('should disable thread pooling for h264 if thread limit is 1', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { threads: 1 } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2458,7 +2478,7 @@ describe(MediaService.name, () => {
});
it('should omit thread flags for h264 if thread limit is at or below 0', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { threads: 0 } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2473,7 +2493,7 @@ describe(MediaService.name, () => {
});
it('should disable thread pooling for hevc if thread limit is 1', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamVp9);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamVp9 });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { threads: 1, targetVideoCodec: VideoCodec.Hevc } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2495,7 +2515,7 @@ describe(MediaService.name, () => {
});
it('should omit thread flags for hevc if thread limit is at or below 0', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamVp9);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamVp9 });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { threads: 0, targetVideoCodec: VideoCodec.Hevc } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2510,7 +2530,7 @@ describe(MediaService.name, () => {
});
it('should use av1 if specified', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamVp9);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamVp9 });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { targetVideoCodec: VideoCodec.Av1 } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2544,7 +2564,7 @@ describe(MediaService.name, () => {
});
it('should map `veryslow` preset to 4 for av1', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamVp9);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamVp9 });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { targetVideoCodec: VideoCodec.Av1, preset: 'veryslow' } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2559,7 +2579,7 @@ describe(MediaService.name, () => {
});
it('should set max bitrate for av1 if specified', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamVp9);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamVp9 });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { targetVideoCodec: VideoCodec.Av1, maxBitrate: '2M' } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2574,7 +2594,7 @@ describe(MediaService.name, () => {
});
it('should set threads for av1 if specified', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamVp9);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamVp9 });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { targetVideoCodec: VideoCodec.Av1, threads: 4 } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2589,7 +2609,7 @@ describe(MediaService.name, () => {
});
it('should set both bitrate and threads for av1 if specified', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamVp9);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamVp9 });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { targetVideoCodec: VideoCodec.Av1, threads: 4, maxBitrate: '2M' },
});
@@ -2606,7 +2626,7 @@ describe(MediaService.name, () => {
});
it('should skip transcoding for audioless videos with optimal policy if video codec is correct', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.noAudioStreams);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.noAudioStreams });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: {
targetVideoCodec: VideoCodec.Hevc,
@@ -2635,15 +2655,15 @@ describe(MediaService.name, () => {
});
});
- it.each(acceptedCodecs)('should skip $codec', async ({ probeStub }) => {
- mocks.media.probe.mockResolvedValue(probeStub);
+ it.each(acceptedCodecs)('should skip $codec', async ({ probeStub: stub }) => {
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...stub });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).not.toHaveBeenCalled();
});
});
it('should use libopus audio encoder when target audio is opus', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.audioStreamAac);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.audioStreamAac });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: {
targetAudioCodec: AudioCodec.Opus,
@@ -2663,7 +2683,7 @@ describe(MediaService.name, () => {
});
it('should fail if hwaccel is enabled for an unsupported codec', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Nvenc, targetVideoCodec: VideoCodec.Vp9 },
});
@@ -2672,14 +2692,14 @@ describe(MediaService.name, () => {
});
it('should fail if hwaccel option is invalid', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { accel: 'invalid' as any } });
await expect(sut.handleVideoConversion({ id: 'video-id' })).rejects.toThrowError();
expect(mocks.media.transcode).not.toHaveBeenCalled();
});
it('should set options for nvenc', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { accel: TranscodeHardwareAcceleration.Nvenc } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2725,7 +2745,7 @@ describe(MediaService.name, () => {
});
it('should set two pass options for nvenc when enabled', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: {
accel: TranscodeHardwareAcceleration.Nvenc,
@@ -2746,7 +2766,7 @@ describe(MediaService.name, () => {
});
it('should set vbr options for nvenc when max bitrate is enabled', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Nvenc, maxBitrate: '10000k' },
});
@@ -2763,7 +2783,7 @@ describe(MediaService.name, () => {
});
it('should set cq options for nvenc when max bitrate is disabled', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Nvenc, maxBitrate: '10000k' },
});
@@ -2780,7 +2800,7 @@ describe(MediaService.name, () => {
});
it('should omit preset for nvenc if invalid', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Nvenc, preset: 'invalid' },
});
@@ -2797,7 +2817,7 @@ describe(MediaService.name, () => {
});
it('should ignore two pass for nvenc if max bitrate is disabled', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { accel: TranscodeHardwareAcceleration.Nvenc } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -2812,7 +2832,7 @@ describe(MediaService.name, () => {
});
it('should use hardware decoding for nvenc if enabled', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Nvenc, accelDecode: true },
});
@@ -2837,7 +2857,7 @@ describe(MediaService.name, () => {
});
it('should use hardware tone-mapping for nvenc if hardware decoding is enabled and should tone map', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamHDR);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamHDR });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Nvenc, accelDecode: true },
});
@@ -2858,7 +2878,7 @@ describe(MediaService.name, () => {
});
it('should set format to nv12 for nvenc if input is not yuv420p', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStream10Bit);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStream10Bit });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Nvenc, accelDecode: true },
});
@@ -2875,7 +2895,7 @@ describe(MediaService.name, () => {
});
it('should set options for qsv', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Qsv, maxBitrate: '10000k' },
});
@@ -2928,7 +2948,7 @@ describe(MediaService.name, () => {
});
it('should set options for qsv with custom dri node', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: {
accel: TranscodeHardwareAcceleration.Qsv,
@@ -2954,7 +2974,7 @@ describe(MediaService.name, () => {
});
it('should omit preset for qsv if invalid', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Qsv, preset: 'invalid' },
});
@@ -2976,7 +2996,7 @@ describe(MediaService.name, () => {
});
it('should set low power mode for qsv if target video codec is vp9', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Qsv, targetVideoCodec: VideoCodec.Vp9 },
});
@@ -2999,7 +3019,7 @@ describe(MediaService.name, () => {
it('should fail for qsv if no hw devices', async () => {
sut.videoInterfaces = { dri: [], mali: false };
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { accel: TranscodeHardwareAcceleration.Qsv } });
await expect(sut.handleVideoConversion({ id: 'video-id' })).rejects.toThrowError();
@@ -3009,7 +3029,7 @@ describe(MediaService.name, () => {
it('should prefer higher index renderD* device for qsv', async () => {
sut.videoInterfaces = { dri: ['card1', 'renderD129', 'card0', 'renderD128'], mali: false };
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { accel: TranscodeHardwareAcceleration.Qsv } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -3029,7 +3049,7 @@ describe(MediaService.name, () => {
});
it('should use hardware decoding for qsv if enabled', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Qsv, accelDecode: true },
});
@@ -3060,7 +3080,7 @@ describe(MediaService.name, () => {
});
it('should use hardware tone-mapping for qsv if hardware decoding is enabled and should tone map', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamHDR);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamHDR });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Qsv, accelDecode: true },
});
@@ -3093,7 +3113,7 @@ describe(MediaService.name, () => {
it('should use preferred device for qsv when hardware decoding', async () => {
sut.videoInterfaces = { dri: ['renderD128', 'renderD129', 'renderD130'], mali: false };
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Qsv, accelDecode: true, preferredHwDevice: 'renderD129' },
});
@@ -3111,7 +3131,7 @@ describe(MediaService.name, () => {
});
it('should set format to nv12 for qsv if input is not yuv420p', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStream10Bit);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStream10Bit });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Qsv, accelDecode: true },
});
@@ -3139,7 +3159,7 @@ describe(MediaService.name, () => {
});
it('should set options for vaapi', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { accel: TranscodeHardwareAcceleration.Vaapi } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -3182,7 +3202,7 @@ describe(MediaService.name, () => {
});
it('should set vbr options for vaapi when max bitrate is enabled', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Vaapi, maxBitrate: '10000k' },
});
@@ -3215,7 +3235,7 @@ describe(MediaService.name, () => {
});
it('should set cq options for vaapi when max bitrate is disabled', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { accel: TranscodeHardwareAcceleration.Vaapi } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -3246,7 +3266,7 @@ describe(MediaService.name, () => {
});
it('should omit preset for vaapi if invalid', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Vaapi, preset: 'invalid' },
});
@@ -3269,7 +3289,7 @@ describe(MediaService.name, () => {
it('should prefer higher index renderD* device for vaapi', async () => {
sut.videoInterfaces = { dri: ['card1', 'renderD129', 'card0', 'renderD128'], mali: false };
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { accel: TranscodeHardwareAcceleration.Vaapi } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -3290,7 +3310,7 @@ describe(MediaService.name, () => {
it('should select specific gpu node if selected', async () => {
sut.videoInterfaces = { dri: ['renderD129', 'card1', 'card0', 'renderD128'], mali: false };
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Vaapi, preferredHwDevice: '/dev/dri/renderD128' },
});
@@ -3312,7 +3332,7 @@ describe(MediaService.name, () => {
});
it('should use hardware decoding for vaapi if enabled', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Vaapi, accelDecode: true },
});
@@ -3341,7 +3361,7 @@ describe(MediaService.name, () => {
});
it('should use hardware tone-mapping for vaapi if hardware decoding is enabled and should tone map', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamHDR);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamHDR });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Vaapi, accelDecode: true },
});
@@ -3371,7 +3391,7 @@ describe(MediaService.name, () => {
});
it('should set format to nv12 for vaapi if input is not yuv420p', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStream10Bit);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStream10Bit });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Vaapi, accelDecode: true },
});
@@ -3398,7 +3418,7 @@ describe(MediaService.name, () => {
it('should use preferred device for vaapi when hardware decoding', async () => {
sut.videoInterfaces = { dri: ['renderD128', 'renderD129', 'renderD130'], mali: false };
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Vaapi, accelDecode: true, preferredHwDevice: 'renderD129' },
});
@@ -3416,7 +3436,7 @@ describe(MediaService.name, () => {
});
it('should fallback to hw encoding and sw decoding if hw transcoding fails and hw decoding is enabled', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Vaapi, accelDecode: true },
});
@@ -3440,7 +3460,7 @@ describe(MediaService.name, () => {
});
it('should fallback to sw decoding if fallback to sw decoding + hw encoding fails', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Vaapi, accelDecode: true },
});
@@ -3460,7 +3480,7 @@ describe(MediaService.name, () => {
});
it('should fallback to sw transcoding if hw transcoding fails and hw decoding is disabled', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { accel: TranscodeHardwareAcceleration.Vaapi } });
mocks.media.transcode.mockRejectedValueOnce(new Error('error'));
await sut.handleVideoConversion({ id: 'video-id' });
@@ -3478,14 +3498,14 @@ describe(MediaService.name, () => {
it('should fail for vaapi if no hw devices', async () => {
sut.videoInterfaces = { dri: [], mali: true };
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { accel: TranscodeHardwareAcceleration.Vaapi } });
await expect(sut.handleVideoConversion({ id: 'video-id' })).rejects.toThrowError();
expect(mocks.media.transcode).not.toHaveBeenCalled();
});
it('should set options for rkmpp', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Rkmpp, accelDecode: true },
});
@@ -3535,7 +3555,7 @@ describe(MediaService.name, () => {
});
it('should set vbr options for rkmpp when max bitrate is enabled', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamVp9);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamVp9 });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: {
accel: TranscodeHardwareAcceleration.Rkmpp,
@@ -3573,7 +3593,7 @@ describe(MediaService.name, () => {
});
it('should set cqp options for rkmpp when max bitrate is disabled', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Rkmpp, accelDecode: true, crf: 30, maxBitrate: '0' },
});
@@ -3606,7 +3626,7 @@ describe(MediaService.name, () => {
});
it('should set OpenCL tonemapping options for rkmpp when OpenCL is available', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamHDR);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamHDR });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Rkmpp, accelDecode: true, crf: 30, maxBitrate: '0' },
});
@@ -3635,7 +3655,7 @@ describe(MediaService.name, () => {
it('should set hardware decoding options for rkmpp when hardware decoding is enabled with no OpenCL on non-HDR file', async () => {
sut.videoInterfaces = { dri: ['renderD128'], mali: false };
- mocks.media.probe.mockResolvedValue(probeStub.noAudioStreams);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.noAudioStreams });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Rkmpp, accelDecode: true, crf: 30, maxBitrate: '0' },
});
@@ -3661,7 +3681,7 @@ describe(MediaService.name, () => {
});
it('should use software decoding and tone-mapping if hardware decoding is disabled', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamHDR);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamHDR });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Rkmpp, accelDecode: false, crf: 30, maxBitrate: '0' },
});
@@ -3683,7 +3703,7 @@ describe(MediaService.name, () => {
it('should use software tone-mapping if opencl is not available', async () => {
sut.videoInterfaces = { dri: ['renderD128'], mali: false };
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamHDR);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamHDR });
mocks.systemMetadata.get.mockResolvedValue({
ffmpeg: { accel: TranscodeHardwareAcceleration.Rkmpp, accelDecode: true, crf: 30, maxBitrate: '0' },
});
@@ -3704,7 +3724,7 @@ describe(MediaService.name, () => {
});
it('should tonemap when policy is required and video is hdr', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamHDR);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamHDR });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.Required } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -3726,7 +3746,7 @@ describe(MediaService.name, () => {
});
it('should tonemap when policy is optimal and video is hdr', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamHDR);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamHDR });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.Optimal } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -3748,7 +3768,7 @@ describe(MediaService.name, () => {
});
it('should transcode when policy is required and video is not yuv420p', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStream10Bit);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStream10Bit });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.Required } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -3763,7 +3783,7 @@ describe(MediaService.name, () => {
});
it('should convert to yuv420p when scaling without tone-mapping', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStream4K10Bit);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStream4K10Bit });
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.Required } });
await sut.handleVideoConversion({ id: 'video-id' });
expect(mocks.media.transcode).toHaveBeenCalledWith(
@@ -3778,38 +3798,30 @@ describe(MediaService.name, () => {
});
it('should count frames for progress when log level is debug', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
mocks.logger.isLevelEnabled.mockReturnValue(true);
await sut.handleVideoConversion({ id: 'video-id' });
- expect(mocks.media.probe).toHaveBeenCalledWith('/original/path.ext', { countFrames: true });
expect(mocks.media.transcode).toHaveBeenCalledWith('/original/path.ext', expect.any(String), {
inputOptions: expect.any(Array),
outputOptions: expect.any(Array),
twoPass: false,
progress: {
- frameCount: probeStub.videoStream2160p.videoStreams[0].frameCount,
+ frameCount: probeStub.videoStream2160p.videoStream!.frameCount,
percentInterval: expect.any(Number),
},
});
});
it('should not count frames for progress when log level is not debug', async () => {
- mocks.media.probe.mockResolvedValue(probeStub.videoStream2160p);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStream2160p });
mocks.logger.isLevelEnabled.mockReturnValue(false);
await sut.handleVideoConversion({ id: 'video-id' });
-
- expect(mocks.media.probe).toHaveBeenCalledWith('/original/path.ext', { countFrames: false });
});
it('should process unknown audio stream', async () => {
- const asset = AssetFactory.create({
- type: AssetType.Video,
- originalPath: '/original/path.ext',
- });
- mocks.media.probe.mockResolvedValue(probeStub.audioStreamUnknown);
- mocks.asset.getByIds.mockResolvedValue([asset]);
+ mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.audioStreamUnknown });
await sut.handleVideoConversion({ id: asset.id });
expect(mocks.media.transcode).toHaveBeenCalledWith(
diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts
index 2c9325c976..0b8a1f6702 100644
--- a/server/src/services/media.service.ts
+++ b/server/src/services/media.service.ts
@@ -15,7 +15,6 @@ import {
ImageFormat,
JobName,
JobStatus,
- LogLevel,
QueueName,
RawExtractedFormat,
StorageFolder,
@@ -506,10 +505,7 @@ export class MediaService extends BaseService {
};
}
- private async generateVideoThumbnails(
- asset: ThumbnailPathEntity & { originalPath: string },
- { ffmpeg, image }: SystemConfig,
- ) {
+ private async generateVideoThumbnails(asset: ThumbnailAsset, { ffmpeg, image }: SystemConfig) {
const previewFile = this.getImageFile(asset, {
fileType: AssetFileType.Preview,
format: image.preview.format,
@@ -526,22 +522,15 @@ export class MediaService extends BaseService {
});
this.storageCore.ensureFolders(previewFile.path);
- const { format, audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath);
- const mainVideoStream = this.getMainStream(videoStreams);
- if (!mainVideoStream) {
- throw new Error(`No video streams found for asset ${asset.id}`);
+ const { videoStream, format } = asset;
+ if (!videoStream || !format) {
+ throw new Error(`Missing video metadata for asset ${asset.id}`);
}
- const mainAudioStream = this.getMainStream(audioStreams);
const previewConfig = ThumbnailConfig.create({ ...ffmpeg, targetResolution: image.preview.size.toString() });
- const thumbnailConfig = ThumbnailConfig.create({ ...ffmpeg, targetResolution: image.thumbnail.size.toString() });
- const previewOptions = previewConfig.getCommand(TranscodeTarget.Video, mainVideoStream, mainAudioStream, format);
- const thumbnailOptions = thumbnailConfig.getCommand(
- TranscodeTarget.Video,
- mainVideoStream,
- mainAudioStream,
- format,
- );
+ const thumbConfig = ThumbnailConfig.create({ ...ffmpeg, targetResolution: image.thumbnail.size.toString() });
+ const previewOptions = previewConfig.getCommand(TranscodeTarget.Video, videoStream, undefined, format ?? undefined);
+ const thumbnailOptions = thumbConfig.getCommand(TranscodeTarget.Video, videoStream, undefined, format ?? undefined);
await this.mediaRepository.transcode(asset.originalPath, previewFile.path, previewOptions);
await this.mediaRepository.transcode(asset.originalPath, thumbnailFile.path, thumbnailOptions);
@@ -554,7 +543,7 @@ export class MediaService extends BaseService {
return {
files: [previewFile, thumbnailFile],
thumbhash,
- fullsizeDimensions: { width: mainVideoStream.width, height: mainVideoStream.height },
+ fullsizeDimensions: { width: videoStream.width, height: videoStream.height },
};
}
@@ -588,17 +577,14 @@ export class MediaService extends BaseService {
const output = StorageCore.getEncodedVideoPath(asset);
this.storageCore.ensureFolders(output);
- const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input, {
- countFrames: this.logger.isLevelEnabled(LogLevel.Debug), // makes frame count more reliable for progress logs
- });
- const videoStream = this.getMainStream(videoStreams);
- const audioStream = this.getMainStream(audioStreams);
- if (!videoStream || !format.formatName) {
+ const { videoStream, format } = asset;
+ const audioStream = asset.audioStream ?? undefined;
+ if (!videoStream || !format) {
+ this.logger.warn(`Skipped transcoding for asset ${asset.id}: missing metadata; re-run extraction first`);
return JobStatus.Failed;
}
-
if (!videoStream.height || !videoStream.width) {
- this.logger.warn(`Skipped transcoding for asset ${asset.id}: no video streams found`);
+ this.logger.warn(`Skipped transcoding for asset ${asset.id}: no video dimensions`);
return JobStatus.Failed;
}
@@ -667,12 +653,6 @@ export class MediaService extends BaseService {
return JobStatus.Success;
}
- private getMainStream<T extends VideoStreamInfo | AudioStreamInfo>(streams: T[]): T {
- return streams
- .filter((stream) => stream.codecName !== 'unknown')
- .toSorted((stream1, stream2) => stream2.bitrate - stream1.bitrate)[0];
- }
-
private getTranscodeTarget(
config: SystemConfigFFmpegDto,
videoStream: VideoStreamInfo,
diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.ts
index b796094bb5..19efd50e99 100644
--- a/server/src/services/metadata.service.spec.ts
+++ b/server/src/services/metadata.service.spec.ts
@@ -18,7 +18,7 @@ import { ImmichTags } from 'src/repositories/metadata.repository';
import { firstDateTime, MetadataService } from 'src/services/metadata.service';
import { AssetFactory } from 'test/factories/asset.factory';
import { PersonFactory } from 'test/factories/person.factory';
-import { probeStub } from 'test/fixtures/media.stub';
+import { videoInfoStub } from 'test/fixtures/media.stub';
import { tagStub } from 'test/fixtures/tag.stub';
import { getForMetadataExtraction, getForSidecarWrite } from 'test/mappers';
import { factory } from 'test/small.factory';
@@ -59,6 +59,15 @@ const makeFaceTags = (face: Partial<{ Name: string }> = {}, orientation?: Immich
},
});
+const emptyPackets = {
+ totalDuration: 0,
+ packetCount: 0,
+ outputFrames: 0,
+ keyframePts: [],
+ keyframeAccDuration: [],
+ keyframeOwnDuration: [],
+};
+
describe(MetadataService.name, () => {
let sut: MetadataService;
let mocks: ServiceMocks;
@@ -183,9 +192,12 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
- expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ dateTimeOriginal: sidecarDate }), {
- lockedPropertiesBehavior: 'skip',
- });
+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
+ expect.objectContaining({
+ exif: expect.objectContaining({ dateTimeOriginal: sidecarDate }),
+ lockedPropertiesBehavior: 'skip',
+ }),
+ );
expect(mocks.asset.update).toHaveBeenCalledWith(
expect.objectContaining({
id: asset.id,
@@ -212,8 +224,10 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
- expect.objectContaining({ dateTimeOriginal: fileModifiedAt }),
- { lockedPropertiesBehavior: 'skip' },
+ expect.objectContaining({
+ exif: expect.objectContaining({ dateTimeOriginal: fileModifiedAt }),
+ lockedPropertiesBehavior: 'skip',
+ }),
);
expect(mocks.asset.update).toHaveBeenCalledWith({
id: asset.id,
@@ -242,8 +256,10 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
- expect.objectContaining({ dateTimeOriginal: fileCreatedAt }),
- { lockedPropertiesBehavior: 'skip' },
+ expect.objectContaining({
+ exif: expect.objectContaining({ dateTimeOriginal: fileCreatedAt }),
+ lockedPropertiesBehavior: 'skip',
+ }),
);
expect(mocks.asset.update).toHaveBeenCalledWith({
id: asset.id,
@@ -265,9 +281,11 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
- dateTimeOriginal: new Date('2022-01-01T00:00:00.000Z'),
+ exif: expect.objectContaining({
+ dateTimeOriginal: new Date('2022-01-01T00:00:00.000Z'),
+ }),
+ lockedPropertiesBehavior: 'skip',
}),
- { lockedPropertiesBehavior: 'skip' },
);
expect(mocks.asset.update).toHaveBeenCalledWith(
@@ -290,9 +308,12 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
- expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ iso: 160 }), {
- lockedPropertiesBehavior: 'skip',
- });
+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
+ expect.objectContaining({
+ exif: expect.objectContaining({ iso: 160 }),
+ lockedPropertiesBehavior: 'skip',
+ }),
+ );
expect(mocks.asset.update).toHaveBeenCalledWith({
id: asset.id,
duration: null,
@@ -323,8 +344,10 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
- expect.objectContaining({ city: null, state: null, country: null }),
- { lockedPropertiesBehavior: 'skip' },
+ expect.objectContaining({
+ exif: expect.objectContaining({ city: null, state: null, country: null }),
+ lockedPropertiesBehavior: 'skip',
+ }),
);
expect(mocks.asset.update).toHaveBeenCalledWith({
id: asset.id,
@@ -353,8 +376,10 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
- expect.objectContaining({ city: 'City', state: 'State', country: 'Country' }),
- { lockedPropertiesBehavior: 'skip' },
+ expect.objectContaining({
+ exif: expect.objectContaining({ city: 'City', state: 'State', country: 'Country' }),
+ lockedPropertiesBehavior: 'skip',
+ }),
);
expect(mocks.asset.update).toHaveBeenCalledWith({
id: asset.id,
@@ -378,8 +403,10 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
- expect.objectContaining({ latitude: null, longitude: null }),
- { lockedPropertiesBehavior: 'skip' },
+ expect.objectContaining({
+ exif: expect.objectContaining({ latitude: null, longitude: null }),
+ lockedPropertiesBehavior: 'skip',
+ }),
);
});
@@ -585,7 +612,7 @@ describe(MetadataService.name, () => {
it('should not apply motion photos if asset is video', async () => {
const asset = AssetFactory.create({ type: AssetType.Video });
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
- mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
+ mocks.media.probe.mockResolvedValue(videoInfoStub.matroskaContainer);
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
@@ -611,15 +638,144 @@ describe(MetadataService.name, () => {
it('should extract the correct video orientation', async () => {
const asset = AssetFactory.create({ type: AssetType.Video });
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamVertical2160p);
+ mocks.media.probe.mockResolvedValue(videoInfoStub.videoStreamVertical2160p);
mockReadTags({});
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
- expect.objectContaining({ orientation: ExifOrientation.Rotate270CW.toString() }),
- { lockedPropertiesBehavior: 'skip' },
+ expect.objectContaining({
+ exif: expect.objectContaining({ orientation: ExifOrientation.Rotate270CW.toString() }),
+ lockedPropertiesBehavior: 'skip',
+ }),
+ );
+ });
+
+ it('should persist CICP smallints and profile/level for HDR10 video', async () => {
+ const asset = AssetFactory.create({ type: AssetType.Video });
+ mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
+ mocks.media.probe.mockResolvedValue(videoInfoStub.videoStreamHDR10);
+ mocks.media.probePackets.mockResolvedValue(emptyPackets);
+ mockReadTags({});
+
+ await sut.handleMetadataExtraction({ id: asset.id });
+
+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
+ expect.objectContaining({
+ exif: expect.objectContaining({ fps: 59.94 }),
+ video: expect.objectContaining({
+ codecName: 'hevc',
+ profile: 2,
+ level: 153,
+ pixelFormat: 'yuv420p10le',
+ colorPrimaries: 9,
+ colorTransfer: 16,
+ colorMatrix: 9,
+ dvProfile: undefined,
+ }),
+ }),
+ );
+ });
+
+ it('should persist Dolby Vision fields', async () => {
+ const asset = AssetFactory.create({ type: AssetType.Video });
+ mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
+ mocks.media.probe.mockResolvedValue(videoInfoStub.videoStreamDolbyVision);
+ mocks.media.probePackets.mockResolvedValue(emptyPackets);
+ mockReadTags({});
+
+ await sut.handleMetadataExtraction({ id: asset.id });
+
+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
+ expect.objectContaining({
+ video: expect.objectContaining({
+ dvProfile: 8,
+ dvLevel: 10,
+ dvBlSignalCompatibilityId: 4,
+ colorTransfer: 18, // ARIB_STD_B67
+ }),
+ }),
+ );
+ });
+
+ it('should persist packet-derived HLS fields', async () => {
+ const asset = AssetFactory.create({ type: AssetType.Video });
+ mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
+ mocks.media.probe.mockResolvedValue(videoInfoStub.videoStreamHDR10);
+ mocks.media.probePackets.mockResolvedValue({
+ totalDuration: 12_080,
+ packetCount: 1148,
+ outputFrames: 1149,
+ keyframePts: [-590, 10, 611, 1211],
+ keyframeAccDuration: [10, 610, 6110, 12_080],
+ keyframeOwnDuration: [10, 10, 10, 10],
+ });
+ mockReadTags({});
+
+ await sut.handleMetadataExtraction({ id: asset.id });
+
+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
+ expect.objectContaining({
+ video: expect.objectContaining({ timeBase: 600 }),
+ keyframes: expect.objectContaining({
+ totalDuration: 12_080,
+ packetCount: 1148,
+ outputFrames: 1149,
+ pts: [-590, 10, 611, 1211],
+ accDuration: [10, 610, 6110, 12_080],
+ ownDuration: [10, 10, 10, 10],
+ }),
+ }),
+ );
+ });
+
+ it('should omit the keyframe row when the probe returns no keyframes', async () => {
+ const asset = AssetFactory.create({ type: AssetType.Video });
+ mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
+ mocks.media.probe.mockResolvedValue(videoInfoStub.videoStreamHDR10);
+ mocks.media.probePackets.mockResolvedValue(emptyPackets);
+ mockReadTags({});
+
+ await sut.handleMetadataExtraction({ id: asset.id });
+
+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
+ expect.not.objectContaining({ keyframes: expect.anything() }),
+ );
+ });
+
+ it('should prefer ffprobe frameRate over exiftool VideoFrameRate', async () => {
+ const asset = AssetFactory.create({ type: AssetType.Video });
+ mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
+ mocks.media.probe.mockResolvedValue(videoInfoStub.videoStreamHDR10);
+ mocks.media.probePackets.mockResolvedValue(emptyPackets);
+ mockReadTags({ VideoFrameRate: '30' });
+
+ await sut.handleMetadataExtraction({ id: asset.id });
+
+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
+ expect.objectContaining({
+ exif: expect.objectContaining({ fps: 59.94 }),
+ lockedPropertiesBehavior: 'skip',
+ }),
+ );
+ });
+
+ it('should not insert audio/video/keyframe rows for image assets', async () => {
+ const asset = AssetFactory.create({ type: AssetType.Image });
+ mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
+ mockReadTags({});
+
+ await sut.handleMetadataExtraction({ id: asset.id });
+
+ expect(mocks.media.probe).not.toHaveBeenCalled();
+ expect(mocks.media.probePackets).not.toHaveBeenCalled();
+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
+ expect.not.objectContaining({
+ audio: expect.anything(),
+ video: expect.anything(),
+ keyframes: expect.anything(),
+ }),
);
});
@@ -909,39 +1065,41 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
- {
- assetId: asset.id,
- bitsPerSample: expect.any(Number),
- autoStackId: null,
- colorspace: tags.ColorSpace,
- dateTimeOriginal: dateForTest,
- description: tags.ImageDescription,
- exifImageHeight: null,
- exifImageWidth: null,
- exposureTime: tags.ExposureTime,
- fNumber: null,
- fileSizeInByte: 123_456,
- focalLength: tags.FocalLength,
- fps: null,
- iso: tags.ISO,
- latitude: null,
- lensModel: tags.LensModel,
- livePhotoCID: tags.MediaGroupUUID,
- longitude: null,
- make: tags.Make,
- model: tags.Model,
- modifyDate: expect.any(Date),
- orientation: tags.Orientation?.toString(),
- profileDescription: tags.ProfileDescription,
- projectionType: 'EQUIRECTANGULAR',
- timeZone: tags.zone,
- rating: tags.Rating,
- country: null,
- state: null,
- city: null,
- tags: ['parent/child'],
- },
- { lockedPropertiesBehavior: 'skip' },
+ expect.objectContaining({
+ exif: {
+ assetId: asset.id,
+ bitsPerSample: expect.any(Number),
+ autoStackId: null,
+ colorspace: tags.ColorSpace,
+ dateTimeOriginal: dateForTest,
+ description: tags.ImageDescription,
+ exifImageHeight: null,
+ exifImageWidth: null,
+ exposureTime: tags.ExposureTime,
+ fNumber: null,
+ fileSizeInByte: 123_456,
+ focalLength: tags.FocalLength,
+ fps: null,
+ iso: tags.ISO,
+ latitude: null,
+ lensModel: tags.LensModel,
+ livePhotoCID: tags.MediaGroupUUID,
+ longitude: null,
+ make: tags.Make,
+ model: tags.Model,
+ modifyDate: expect.any(Date),
+ orientation: tags.Orientation?.toString(),
+ profileDescription: tags.ProfileDescription,
+ projectionType: 'EQUIRECTANGULAR',
+ timeZone: tags.zone,
+ rating: tags.Rating,
+ country: null,
+ state: null,
+ city: null,
+ tags: ['parent/child'],
+ },
+ lockedPropertiesBehavior: 'skip',
+ }),
);
expect(mocks.asset.update).toHaveBeenCalledWith(
expect.objectContaining({
@@ -975,9 +1133,11 @@ describe(MetadataService.name, () => {
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
- timeZone: 'UTC+0',
+ exif: expect.objectContaining({
+ timeZone: 'UTC+0',
+ }),
+ lockedPropertiesBehavior: 'skip',
}),
- { lockedPropertiesBehavior: 'skip' },
);
});
@@ -985,9 +1145,9 @@ describe(MetadataService.name, () => {
const asset = AssetFactory.create({ type: AssetType.Video });
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mocks.media.probe.mockResolvedValue({
- ...probeStub.videoStreamH264,
+ ...videoInfoStub.videoStreamH264,
format: {
- ...probeStub.videoStreamH264.format,
+ ...videoInfoStub.videoStreamH264.format,
duration: 6.21,
},
});
@@ -1008,9 +1168,9 @@ describe(MetadataService.name, () => {
const asset = AssetFactory.create();
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mocks.media.probe.mockResolvedValue({
- ...probeStub.videoStreamH264,
+ ...videoInfoStub.videoStreamH264,
format: {
- ...probeStub.videoStreamH264.format,
+ ...videoInfoStub.videoStreamH264.format,
duration: 6.21,
},
});
@@ -1030,9 +1190,9 @@ describe(MetadataService.name, () => {
const asset = AssetFactory.create({ type: AssetType.Video });
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mocks.media.probe.mockResolvedValue({
- ...probeStub.videoStreamH264,
+ ...videoInfoStub.videoStreamH264,
format: {
- ...probeStub.videoStreamH264.format,
+ ...videoInfoStub.videoStreamH264.format,
duration: 0,
},
});
@@ -1053,9 +1213,9 @@ describe(MetadataService.name, () => {
const asset = AssetFactory.create({ type: AssetType.Video });
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mocks.media.probe.mockResolvedValue({
- ...probeStub.videoStreamH264,
+ ...videoInfoStub.videoStreamH264,
format: {
- ...probeStub.videoStreamH264.format,
+ ...videoInfoStub.videoStreamH264.format,
duration: 604_800,
},
});
@@ -1111,9 +1271,9 @@ describe(MetadataService.name, () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mockReadTags({ Duration: 123 }, {});
mocks.media.probe.mockResolvedValue({
- ...probeStub.videoStreamH264,
+ ...videoInfoStub.videoStreamH264,
format: {
- ...probeStub.videoStreamH264.format,
+ ...videoInfoStub.videoStreamH264.format,
duration: 456,
},
});
@@ -1132,18 +1292,22 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
- description: '',
+ exif: expect.objectContaining({
+ description: '',
+ }),
+ lockedPropertiesBehavior: 'skip',
}),
- { lockedPropertiesBehavior: 'skip' },
);
mockReadTags({ ImageDescription: ' my\n description' });
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
- description: 'my\n description',
+ exif: expect.objectContaining({
+ description: 'my\n description',
+ }),
+ lockedPropertiesBehavior: 'skip',
}),
- { lockedPropertiesBehavior: 'skip' },
);
});
@@ -1155,9 +1319,11 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
- description: '1000',
+ exif: expect.objectContaining({
+ description: '1000',
+ }),
+ lockedPropertiesBehavior: 'skip',
}),
- { lockedPropertiesBehavior: 'skip' },
);
});
@@ -1388,9 +1554,11 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
- modifyDate: expect.any(Date),
+ exif: expect.objectContaining({
+ modifyDate: expect.any(Date),
+ }),
+ lockedPropertiesBehavior: 'skip',
}),
- { lockedPropertiesBehavior: 'skip' },
);
});
@@ -1402,9 +1570,11 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
- rating: null,
+ exif: expect.objectContaining({
+ rating: null,
+ }),
+ lockedPropertiesBehavior: 'skip',
}),
- { lockedPropertiesBehavior: 'skip' },
);
});
@@ -1416,9 +1586,11 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
- rating: 5,
+ exif: expect.objectContaining({
+ rating: 5,
+ }),
+ lockedPropertiesBehavior: 'skip',
}),
- { lockedPropertiesBehavior: 'skip' },
);
});
@@ -1430,9 +1602,11 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
- rating: null,
+ exif: expect.objectContaining({
+ rating: null,
+ }),
+ lockedPropertiesBehavior: 'skip',
}),
- { lockedPropertiesBehavior: 'skip' },
);
});
@@ -1444,9 +1618,11 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
- rating: -1,
+ exif: expect.objectContaining({
+ rating: -1,
+ }),
+ lockedPropertiesBehavior: 'skip',
}),
- { lockedPropertiesBehavior: 'skip' },
);
});
@@ -1466,7 +1642,7 @@ describe(MetadataService.name, () => {
it('should handle not finding a match', async () => {
const asset = AssetFactory.create({ type: AssetType.Video });
- mocks.media.probe.mockResolvedValue(probeStub.videoStreamVertical2160p);
+ mocks.media.probe.mockResolvedValue(videoInfoStub.videoStreamVertical2160p);
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
mockReadTags({ ContentIdentifier: 'CID' });
@@ -1578,9 +1754,12 @@ describe(MetadataService.name, () => {
mockReadTags(exif);
await sut.handleMetadataExtraction({ id: asset.id });
- expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining(expected), {
- lockedPropertiesBehavior: 'skip',
- });
+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
+ expect.objectContaining({
+ exif: expect.objectContaining(expected),
+ lockedPropertiesBehavior: 'skip',
+ }),
+ );
});
it.each([
@@ -1605,9 +1784,11 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({
- lensModel: expected,
+ exif: expect.objectContaining({
+ lensModel: expected,
+ }),
+ lockedPropertiesBehavior: 'skip',
}),
- { lockedPropertiesBehavior: 'skip' },
);
});
diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts
index 168f7634fd..a3e9c19472 100644
--- a/server/src/services/metadata.service.ts
+++ b/server/src/services/metadata.service.ts
@@ -243,10 +243,11 @@ export class MetadataService extends BaseService {
return;
}
- const [exifTags, stats] = await Promise.all([
+ const [exifResult, stats] = await Promise.all([
this.getExifTags(asset),
this.storageRepository.stat(asset.originalPath),
]);
+ const { tags: exifTags, audio, video, packets, format } = exifResult;
this.logger.verbose('Exif Tags', exifTags);
const dates = this.getDates(asset, exifTags, stats);
@@ -294,7 +295,7 @@ export class MetadataService extends BaseService {
exifTags.Make ?? exifTags.Device?.Manufacturer ?? exifTags.AndroidMake ?? (exifTags.DeviceManufacturer || null),
model:
exifTags.Model ?? exifTags.Device?.ModelName ?? exifTags.AndroidModel ?? (exifTags.DeviceModelName || null),
- fps: validate(Number.parseFloat(exifTags.VideoFrameRate!)),
+ fps: video?.frameRate ?? validate(Number.parseFloat(exifTags.VideoFrameRate!)),
iso: validate(exifTags.ISO) as number,
exposureTime: exifTags.ExposureTime ?? null,
lensModel: getLensModel(exifTags),
@@ -313,6 +314,53 @@ export class MetadataService extends BaseService {
tags: tags.length > 0 ? tags : null,
};
+ const audioData =
+ format && audio?.codecName
+ ? {
+ assetId: asset.id,
+ bitrate: audio.bitrate,
+ index: audio.index,
+ profile: audio.profile,
+ codecName: audio.codecName,
+ }
+ : undefined;
+
+ const videoData =
+ format?.formatName && format?.formatLongName && video?.codecName && video?.timeBase
+ ? {
+ assetId: asset.id,
+ bitrate: video.bitrate,
+ frameCount: video.frameCount,
+ timeBase: video.timeBase,
+ index: video.index,
+ profile: video.profile,
+ level: video.level,
+ colorPrimaries: video.colorPrimaries,
+ colorTransfer: video.colorTransfer,
+ colorMatrix: video.colorMatrix,
+ dvProfile: video.dvProfile,
+ dvLevel: video.dvLevel,
+ dvBlSignalCompatibilityId: video.dvBlSignalCompatibilityId,
+ codecName: video.codecName,
+ formatName: format.formatName,
+ formatLongName: format.formatLongName,
+ pixelFormat: video.pixelFormat,
+ }
+ : undefined;
+
+ const keyframeData =
+ packets && packets.keyframePts.length > 0
+ ? {
+ assetId: asset.id,
+ totalDuration: packets.totalDuration,
+ packetCount: packets.packetCount,
+ outputFrames: packets.outputFrames,
+ pts: packets.keyframePts,
+ accDuration: packets.keyframeAccDuration,
+ ownDuration: packets.keyframeOwnDuration,
+ }
+ : undefined;
+
const isSidewards = exifTags.Orientation && this.isOrientationSidewards(exifTags.Orientation);
const assetWidth = isSidewards ? validate(height) : validate(width);
const assetHeight = isSidewards ? validate(width) : validate(height);
@@ -333,7 +381,13 @@ export class MetadataService extends BaseService {
height: !asset.isEdited || asset.height == null ? assetHeight : undefined,
}),
async () => {
- await this.assetRepository.upsertExif(exifData, { lockedPropertiesBehavior: 'skip' });
+ await this.assetRepository.upsertExif({
+ exif: exifData,
+ audio: audioData,
+ video: videoData,
+ keyframes: keyframeData,
+ lockedPropertiesBehavior: 'skip',
+ });
await this.applyTagList(asset);
},
);
@@ -523,13 +577,14 @@ export class MetadataService extends BaseService {
return { width, height };
}
- private async getExifTags(asset: { originalPath: string; files: AssetFile[]; type: AssetType }): Promise<ImmichTags> {
+ private async getExifTags(asset: { originalPath: string; files: AssetFile[]; type: AssetType }) {
const { sidecarFile } = getAssetFiles(asset.files);
+ const shouldProbe = asset.type === AssetType.Video || asset.originalPath.toLowerCase().endsWith('.gif');
- const [mediaTags, sidecarTags, videoTags] = await Promise.all([
+ const [mediaTags, sidecarTags, videoResult] = await Promise.all([
this.metadataRepository.readTags(asset.originalPath),
sidecarFile ? this.metadataRepository.readTags(sidecarFile.path) : null,
- asset.type === AssetType.Video ? this.getVideoTags(asset.originalPath) : null,
+ shouldProbe ? this.getVideoTags(asset.originalPath) : null,
]);
// prefer dates from sidecar tags
@@ -554,14 +609,20 @@ export class MetadataService extends BaseService {
// prefer duration from video tags
// don't save duration if asset is definitely not an animated image (see e.g. CR3 with Duration: 1s)
- if (videoTags || !mimeTypes.isPossiblyAnimatedImage(asset.originalPath)) {
+ if (videoResult || !mimeTypes.isPossiblyAnimatedImage(asset.originalPath)) {
delete mediaTags.Duration;
}
// never use duration from sidecar
delete sidecarTags?.Duration;
- return { ...mediaTags, ...videoTags, ...sidecarTags };
+ return {
+ tags: { ...mediaTags, ...videoResult?.tags, ...sidecarTags },
+ audio: videoResult?.audio,
+ video: videoResult?.video,
+ packets: videoResult?.packets,
+ format: videoResult?.format ?? null,
+ };
}
private getTagList(exifTags: ImmichTags): string[] {
@@ -1008,20 +1069,22 @@ export class MetadataService extends BaseService {
}
private async getVideoTags(originalPath: string) {
- const { videoStreams, format } = await this.mediaRepository.probe(originalPath);
+ const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(originalPath);
+ const video = videoStreams[0];
+ const audio = audioStreams[0];
+ const packets = video?.timeBase ? await this.mediaRepository.probePackets(originalPath, video.index) : null;
const tags: Pick<ImmichTags, 'Duration' | 'Orientation' | 'ImageWidth' | 'ImageHeight'> = {};
- if (videoStreams[0]) {
- // Set video dimensions
- if (videoStreams[0].width) {
- tags.ImageWidth = videoStreams[0].width;
+ if (video) {
+ if (video.width) {
+ tags.ImageWidth = video.width;
}
- if (videoStreams[0].height) {
- tags.ImageHeight = videoStreams[0].height;
+ if (video.height) {
+ tags.ImageHeight = video.height;
}
- switch (videoStreams[0].rotation) {
+ switch (video.rotation) {
case -90: {
tags.Orientation = ExifOrientation.Rotate90CW;
break;
@@ -1045,6 +1108,6 @@ export class MetadataService extends BaseService {
tags.Duration = format.duration;
}
- return tags;
+ return { tags, audio, video, packets, format };
}
}
diff --git a/server/src/services/tag.service.spec.ts b/server/src/services/tag.service.spec.ts
index 6fc472bb87..0c748fded8 100644
--- a/server/src/services/tag.service.spec.ts
+++ b/server/src/services/tag.service.spec.ts
@@ -206,16 +206,22 @@ describe(TagService.name, () => {
count: 6,
});
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
- { assetId: 'asset-1', lockedProperties: ['tags'], tags: ['tag-1', 'tag-2'] },
- { lockedPropertiesBehavior: 'append' },
+ expect.objectContaining({
+ exif: { assetId: 'asset-1', lockedProperties: ['tags'], tags: ['tag-1', 'tag-2'] },
+ lockedPropertiesBehavior: 'append',
+ }),
);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
- { assetId: 'asset-2', lockedProperties: ['tags'], tags: ['tag-1', 'tag-2'] },
- { lockedPropertiesBehavior: 'append' },
+ expect.objectContaining({
+ exif: { assetId: 'asset-2', lockedProperties: ['tags'], tags: ['tag-1', 'tag-2'] },
+ lockedPropertiesBehavior: 'append',
+ }),
);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
- { assetId: 'asset-3', lockedProperties: ['tags'], tags: ['tag-1', 'tag-2'] },
- { lockedPropertiesBehavior: 'append' },
+ expect.objectContaining({
+ exif: { assetId: 'asset-3', lockedProperties: ['tags'], tags: ['tag-1', 'tag-2'] },
+ lockedPropertiesBehavior: 'append',
+ }),
);
expect(mocks.tag.upsertAssetIds).toHaveBeenCalledWith([
{ tagId: 'tag-1', assetId: 'asset-1' },
@@ -255,12 +261,16 @@ describe(TagService.name, () => {
]);
expect(mocks.asset.upsertExif).not.toHaveBeenCalledWith(
- { assetId: 'asset-1', lockedProperties: ['tags'], tags: ['tag-1'] },
- { lockedPropertiesBehavior: 'append' },
+ expect.objectContaining({
+ exif: { assetId: 'asset-1', lockedProperties: ['tags'], tags: ['tag-1'] },
+ lockedPropertiesBehavior: 'append',
+ }),
);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
- { assetId: 'asset-2', lockedProperties: ['tags'], tags: ['tag-1'] },
- { lockedPropertiesBehavior: 'append' },
+ expect.objectContaining({
+ exif: { assetId: 'asset-2', lockedProperties: ['tags'], tags: ['tag-1'] },
+ lockedPropertiesBehavior: 'append',
+ }),
);
expect(mocks.tag.getAssetIds).toHaveBeenCalledWith('tag-1', ['asset-1', 'asset-2']);
expect(mocks.tag.addAssetIds).toHaveBeenCalledWith('tag-1', ['asset-2']);
diff --git a/server/src/services/tag.service.ts b/server/src/services/tag.service.ts
index d34cd84ecd..8b92d3abf8 100644
--- a/server/src/services/tag.service.ts
+++ b/server/src/services/tag.service.ts
@@ -152,7 +152,8 @@ export class TagService extends BaseService {
private async updateTags(assetId: string) {
const { tags } = await this.assetRepository.getForUpdateTags(assetId);
- await this.assetRepository.upsertExif(updateLockedColumns({ assetId, tags: tags.map(({ value }) => value) }), {
+ await this.assetRepository.upsertExif({
+ exif: updateLockedColumns({ assetId, tags: tags.map(({ value }) => value) }),
lockedPropertiesBehavior: 'append',
});
}
diff --git a/server/src/types.ts b/server/src/types.ts
index c33b5a18ad..86ba0a1cc2 100644
--- a/server/src/types.ts
+++ b/server/src/types.ts
@@ -7,9 +7,18 @@ import { AuthDto } from 'src/dtos/auth.dto';
import { AssetEditActionItem } from 'src/dtos/editing.dto';
import { SetMaintenanceModeDto } from 'src/dtos/maintenance.dto';
import {
+ AacProfile,
AssetOrder,
AssetType,
+ Av1Profile,
+ ColorMatrix,
+ ColorPrimaries,
+ ColorTransfer,
+ DvProfile,
+ DvSignalCompatibility,
ExifOrientation,
+ H264Profile,
+ HevcProfile,
ImageFormat,
JobName,
MemoryType,
@@ -81,21 +90,44 @@ export interface VideoStreamInfo {
width: number;
rotation: number;
codecName?: string;
+ profile?: H264Profile | HevcProfile | Av1Profile;
+ level?: number;
frameCount: number;
- isHDR: boolean;
+ frameRate?: number;
+ timeBase?: number;
bitrate: number;
pixelFormat: string;
- colorPrimaries?: string;
- colorSpace?: string;
- colorTransfer?: string;
+ colorPrimaries: ColorPrimaries;
+ colorMatrix: ColorMatrix;
+ colorTransfer: ColorTransfer;
+ dvProfile?: DvProfile;
+ dvLevel?: number;
+ dvBlSignalCompatibilityId?: DvSignalCompatibility;
}
export interface AudioStreamInfo {
index: number;
codecName?: string;
+ profile?: AacProfile;
bitrate: number;
}
+/** Packet-derived video data needed for accurate HLS playlists. */
+export interface VideoPacketInfo {
+ /** Sum of source packet duration across all packets (includes discard). */
+ totalDuration: number;
+ /** Post-discard packet count. */
+ packetCount: number;
+ /** Output CFR frame count at `packetCount / format.duration`. */
+ outputFrames: number;
+ /** All keyframe PTS in source ticks, including pre-roll discard keyframes. */
+ keyframePts: number[];
+ /** Cumulative packet duration through each keyframe, inclusive. */
+ keyframeAccDuration: number[];
+ /** Each keyframe's own packet duration (needed for VFR). */
+ keyframeOwnDuration: number[];
+}
+
export interface VideoFormat {
formatName?: string;
formatLongName?: string;
@@ -144,7 +176,7 @@ export interface VideoCodecSWConfig {
getCommand(
target: TranscodeTarget,
videoStream: VideoStreamInfo,
- audioStream: AudioStreamInfo,
+ audioStream?: AudioStreamInfo,
format?: VideoFormat,
): TranscodeCommand;
}
diff --git a/server/src/utils/database.ts b/server/src/utils/database.ts
index 837aa9c8d5..fdf1d98caf 100644
--- a/server/src/utils/database.ts
+++ b/server/src/utils/database.ts
@@ -17,11 +17,11 @@ import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres';
import { Notice, PostgresError } from 'postgres';
import { columns, lockableProperties, LockableProperty, Person } from 'src/database';
import { AssetEditActionItem } from 'src/dtos/editing.dto';
-import { AssetFileType, AssetVisibility, DatabaseExtension } from 'src/enum';
+import { AssetFileType, AssetVisibility, DatabaseExtension, ExifOrientation } from 'src/enum';
import { AssetSearchBuilderOptions } from 'src/repositories/search.repository';
import { DB } from 'src/schema';
import { AssetExifTable } from 'src/schema/tables/asset-exif.table';
-import { VectorExtension } from 'src/types';
+import { AudioStreamInfo, VectorExtension, VideoFormat, VideoStreamInfo } from 'src/types';
export const getKyselyConfig = (connection: DatabaseConnectionParams): KyselyConfig => {
return {
@@ -99,6 +99,65 @@ export function withExifInner<O>(qb: SelectQueryBuilder<DB, 'asset', O>) {
.$narrowType<{ exifInfo: NotNull }>();
}
+export const dummy = sql`(select 1)`.as('dummy');
+
+export function withAudioStream(eb: ExpressionBuilder<DB, 'asset_exif' | 'asset_audio'>) {
+ return jsonObjectFrom(
+ eb
+ .selectFrom(dummy)
+ .select(['asset_audio.index', 'asset_audio.codecName', 'asset_audio.profile', 'asset_audio.bitrate'])
+ .where('asset_audio.assetId', 'is not', sql.lit(null))
+ .$castTo<AudioStreamInfo | null>(),
+ );
+}
+
+export function withVideoStream(eb: ExpressionBuilder<DB, 'asset_exif' | 'asset_video'>) {
+ return jsonObjectFrom(
+ eb
+ .selectFrom(dummy)
+ .select((eb) => [
+ 'asset_video.index',
+ 'asset_video.codecName',
+ 'asset_video.profile',
+ 'asset_video.level',
+ 'asset_video.bitrate',
+ 'asset_exif.exifImageWidth as width',
+ 'asset_exif.exifImageHeight as height',
+ 'asset_video.pixelFormat',
+ 'asset_video.frameCount',
+ 'asset_exif.fps as frameRate',
+ 'asset_video.timeBase',
+ eb
+ .case()
+ .when('asset_exif.orientation', '=', sql.lit(ExifOrientation.Rotate90CW.toString()))
+ .then(sql.lit(-90))
+ .when('asset_exif.orientation', '=', sql.lit(ExifOrientation.Rotate270CW.toString()))
+ .then(sql.lit(90))
+ .when('asset_exif.orientation', '=', sql.lit(ExifOrientation.Rotate180.toString()))
+ .then(sql.lit(180))
+ .else(0)
+ .end()
+ .as('rotation'),
+ 'asset_video.colorPrimaries',
+ 'asset_video.colorMatrix',
+ 'asset_video.colorTransfer',
+ 'asset_video.dvProfile',
+ 'asset_video.dvLevel',
+ 'asset_video.dvBlSignalCompatibilityId',
+ ])
+ .where('asset_video.assetId', 'is not', sql.lit(null)),
+ ).$castTo<(VideoStreamInfo & { timeBase: NotNull }) | null>();
+}
+
+export function withVideoFormat(eb: ExpressionBuilder<DB, 'asset' | 'asset_video'>) {
+ return jsonObjectFrom(
+ eb
+ .selectFrom(dummy)
+ .select(['asset_video.formatName', 'asset_video.formatLongName', 'asset.duration', 'asset_video.bitrate'])
+ .where('asset_video.assetId', 'is not', sql.lit(null)),
+ ).$castTo<VideoFormat | null>();
+}
+
export function withSmartSearch<O>(qb: SelectQueryBuilder<DB, 'asset', O>) {
return qb
.leftJoin('smart_search', 'asset.id', 'smart_search.assetId')
@@ -445,5 +504,3 @@ export const updateLockedColumns = <T extends Record<string, unknown> & { locked
exif.lockedProperties = lockableProperties.filter((property) => property in exif);
return exif;
};
-
-export const dummy = sql`(select 1)`.as('dummy');
diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts
index 5525dafe94..49e11edab7 100644
--- a/server/src/utils/media.ts
+++ b/server/src/utils/media.ts
@@ -1,6 +1,15 @@
import { AUDIO_ENCODER } from 'src/constants';
import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto';
-import { CQMode, ToneMapping, TranscodeHardwareAcceleration, TranscodeTarget, VideoCodec } from 'src/enum';
+import {
+ ColorMatrix,
+ ColorPrimaries,
+ ColorTransfer,
+ CQMode,
+ ToneMapping,
+ TranscodeHardwareAcceleration,
+ TranscodeTarget,
+ VideoCodec,
+} from 'src/enum';
import {
AudioStreamInfo,
BitrateDistribution,
@@ -255,7 +264,10 @@ export class BaseConfig implements VideoCodecSWConfig {
}
shouldToneMap(videoStream: VideoStreamInfo) {
- return videoStream.isHDR && this.config.tonemap !== ToneMapping.Disabled;
+ return (
+ this.config.tonemap !== ToneMapping.Disabled &&
+ (videoStream.colorTransfer === ColorTransfer.Smpte2084 || videoStream.colorTransfer === ColorTransfer.AribStdB67)
+ );
}
getScaling(videoStream: VideoStreamInfo, mult = 2) {
@@ -409,15 +421,15 @@ export class ThumbnailConfig extends BaseConfig {
: ['-skip_frame', 'nointra', '-sws_flags', 'accurate_rnd+full_chroma_int'];
const metadataOverrides = [];
- if (videoStream.colorPrimaries === 'reserved') {
+ if (videoStream.colorPrimaries === ColorPrimaries.Reserved) {
metadataOverrides.push('colour_primaries=1');
}
- if (videoStream.colorSpace === 'reserved') {
+ if (videoStream.colorMatrix === ColorMatrix.Reserved) {
metadataOverrides.push('matrix_coefficients=1');
}
- if (videoStream.colorTransfer === 'reserved') {
+ if (videoStream.colorTransfer === ColorTransfer.Reserved) {
metadataOverrides.push('transfer_characteristics=1');
}
diff --git a/server/test/fixtures/media.stub.ts b/server/test/fixtures/media.stub.ts
index 23617fcaf0..120fe07664 100644
--- a/server/test/fixtures/media.stub.ts
+++ b/server/test/fixtures/media.stub.ts
@@ -1,3 +1,5 @@
+import { NotNull } from 'kysely';
+import { ColorMatrix, ColorPrimaries, ColorTransfer, DvProfile, DvSignalCompatibility } from 'src/enum';
import { AudioStreamInfo, VideoFormat, VideoInfo, VideoStreamInfo } from 'src/types';
const probeStubDefaultFormat: VideoFormat = {
@@ -15,9 +17,12 @@ const probeStubDefaultVideoStream: VideoStreamInfo[] = [
codecName: 'hevc',
frameCount: 100,
rotation: 0,
- isHDR: false,
bitrate: 0,
+ colorPrimaries: ColorPrimaries.Bt709,
+ colorTransfer: ColorTransfer.Bt709,
+ colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
+ timeBase: 600,
},
];
@@ -29,33 +34,40 @@ const probeStubDefault: VideoInfo = {
audioStreams: probeStubDefaultAudioStream,
};
-export const probeStub = {
+/** Fixtures in the shape `mediaRepository.probe()` returns (arrays of streams, raw ffprobe format). */
+export const videoInfoStub = {
noVideoStreams: Object.freeze<VideoInfo>({ ...probeStubDefault, videoStreams: [] }),
noAudioStreams: Object.freeze<VideoInfo>({ ...probeStubDefault, audioStreams: [] }),
multipleVideoStreams: Object.freeze<VideoInfo>({
...probeStubDefault,
videoStreams: [
{
- index: 0,
+ index: 1,
height: 1080,
width: 400,
codecName: 'hevc',
- frameCount: 1,
+ frameCount: 2,
rotation: 0,
- isHDR: false,
- bitrate: 100,
+ bitrate: 101,
+ colorPrimaries: ColorPrimaries.Bt709,
+ colorTransfer: ColorTransfer.Bt709,
+ colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
+ timeBase: 600,
},
{
- index: 1,
+ index: 0,
height: 1080,
width: 400,
codecName: 'hevc',
- frameCount: 2,
+ frameCount: 1,
rotation: 0,
- isHDR: false,
- bitrate: 101,
+ bitrate: 100,
+ colorPrimaries: ColorPrimaries.Bt709,
+ colorTransfer: ColorTransfer.Bt709,
+ colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
+ timeBase: 600,
},
{
index: 2,
@@ -64,18 +76,21 @@ export const probeStub = {
codecName: 'h7000',
frameCount: 3,
rotation: 0,
- isHDR: false,
bitrate: 99,
+ colorPrimaries: ColorPrimaries.Bt709,
+ colorTransfer: ColorTransfer.Bt709,
+ colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
+ timeBase: 600,
},
],
}),
multipleAudioStreams: Object.freeze<VideoInfo>({
...probeStubDefault,
audioStreams: [
- { index: 0, codecName: 'mp3', bitrate: 100 },
- { index: 1, codecName: 'mp3', bitrate: 101 },
{ index: 2, codecName: 'mp3', bitrate: 102 },
+ { index: 1, codecName: 'mp3', bitrate: 101 },
+ { index: 0, codecName: 'mp3', bitrate: 100 },
],
}),
noHeight: Object.freeze<VideoInfo>({
@@ -88,9 +103,12 @@ export const probeStub = {
codecName: 'hevc',
frameCount: 100,
rotation: 0,
- isHDR: false,
bitrate: 0,
+ colorPrimaries: ColorPrimaries.Bt709,
+ colorTransfer: ColorTransfer.Bt709,
+ colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
+ timeBase: 600,
},
],
}),
@@ -104,9 +122,12 @@ export const probeStub = {
codecName: 'h264',
frameCount: 100,
rotation: 0,
- isHDR: false,
bitrate: 0,
+ colorPrimaries: ColorPrimaries.Bt709,
+ colorTransfer: ColorTransfer.Bt709,
+ colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
+ timeBase: 600,
},
],
}),
@@ -117,8 +138,10 @@ export const probeStub = {
videoStreamMTS: Object.freeze<VideoInfo>({
...probeStubDefault,
format: {
- ...probeStubDefaultFormat,
formatName: 'mpegts',
+ formatLongName: 'MPEG-TS (MPEG-2 Transport Stream)',
+ duration: 0,
+ bitrate: 0,
},
}),
videoStreamHDR: Object.freeze<VideoInfo>({
@@ -131,9 +154,12 @@ export const probeStub = {
codecName: 'h264',
frameCount: 100,
rotation: 0,
- isHDR: true,
+ colorPrimaries: ColorPrimaries.Bt2020,
+ colorMatrix: ColorMatrix.Bt2020Nc,
+ colorTransfer: ColorTransfer.Smpte2084,
bitrate: 0,
pixelFormat: 'yuv420p10le',
+ timeBase: 600,
},
],
}),
@@ -147,9 +173,12 @@ export const probeStub = {
codecName: 'h264',
frameCount: 100,
rotation: 0,
- isHDR: false,
bitrate: 0,
+ colorPrimaries: ColorPrimaries.Bt709,
+ colorTransfer: ColorTransfer.Bt709,
+ colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p10le',
+ timeBase: 600,
},
],
}),
@@ -163,9 +192,12 @@ export const probeStub = {
codecName: 'h264',
frameCount: 100,
rotation: 0,
- isHDR: false,
bitrate: 0,
+ colorPrimaries: ColorPrimaries.Bt709,
+ colorTransfer: ColorTransfer.Bt709,
+ colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p10le',
+ timeBase: 600,
},
],
}),
@@ -179,9 +211,12 @@ export const probeStub = {
codecName: 'h264',
frameCount: 100,
rotation: 90,
- isHDR: false,
bitrate: 0,
+ colorPrimaries: ColorPrimaries.Bt709,
+ colorTransfer: ColorTransfer.Bt709,
+ colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
+ timeBase: 600,
},
],
}),
@@ -195,9 +230,12 @@ export const probeStub = {
codecName: 'h264',
frameCount: 100,
rotation: 0,
- isHDR: false,
bitrate: 0,
+ colorPrimaries: ColorPrimaries.Bt709,
+ colorTransfer: ColorTransfer.Bt709,
+ colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
+ timeBase: 600,
},
],
}),
@@ -211,9 +249,12 @@ export const probeStub = {
codecName: 'h264',
frameCount: 100,
rotation: 0,
- isHDR: false,
bitrate: 0,
+ colorPrimaries: ColorPrimaries.Bt709,
+ colorTransfer: ColorTransfer.Bt709,
+ colorMatrix: ColorMatrix.Bt709,
pixelFormat: 'yuv420p',
+ timeBase: 600,
},
],
}),
@@ -274,10 +315,95 @@ export const probeStub = {
videoStreams: [
{
...probeStubDefaultVideoStream[0],
- colorPrimaries: 'reserved',
- colorSpace: 'reserved',
- colorTransfer: 'reserved',
+ colorPrimaries: ColorPrimaries.Reserved,
+ colorMatrix: ColorMatrix.Reserved,
+ colorTransfer: ColorTransfer.Reserved,
+ },
+ ],
+ }),
+ videoStreamHDR10: Object.freeze<VideoInfo>({
+ ...probeStubDefault,
+ videoStreams: [
+ {
+ index: 0,
+ height: 2160,
+ width: 3840,
+ codecName: 'hevc',
+ profile: 2,
+ level: 153,
+ frameCount: 1208,
+ frameRate: 59.94,
+ rotation: 0,
+ bitrate: 64_000_000,
+ pixelFormat: 'yuv420p10le',
+ colorPrimaries: ColorPrimaries.Bt2020,
+ colorMatrix: ColorMatrix.Bt2020Nc,
+ colorTransfer: ColorTransfer.Smpte2084,
+ timeBase: 600,
+ },
+ ],
+ }),
+ videoStreamDolbyVision: Object.freeze<VideoInfo>({
+ ...probeStubDefault,
+ videoStreams: [
+ {
+ index: 0,
+ height: 2160,
+ width: 3840,
+ codecName: 'hevc',
+ profile: 2,
+ level: 153,
+ frameCount: 1299,
+ frameRate: 59.94,
+ rotation: 0,
+ bitrate: 53_500_000,
+ pixelFormat: 'yuv420p10le',
+ colorPrimaries: ColorPrimaries.Bt2020,
+ colorMatrix: ColorMatrix.Bt2020Nc,
+ colorTransfer: ColorTransfer.AribStdB67,
+ dvProfile: DvProfile.Dvhe08,
+ dvLevel: 10,
+ dvBlSignalCompatibilityId: DvSignalCompatibility.Hlg,
+ timeBase: 600,
+ },
+ ],
+ }),
+ videoStreamWithProfileLevel: Object.freeze<VideoInfo>({
+ ...probeStubDefault,
+ videoStreams: [
+ {
+ ...probeStubDefaultVideoStream[0],
+ codecName: 'h264',
+ profile: 100,
+ level: 40,
+ },
+ ],
+ }),
+ audioStreamAAC: Object.freeze<VideoInfo>({
+ ...probeStubDefault,
+ audioStreams: [
+ {
+ index: 1,
+ codecName: 'aac',
+ profile: 2,
+ bitrate: 128_000,
},
],
}),
};
+
+interface SelectedStreams {
+ videoStream: VideoStreamInfo & { timeBase: NotNull };
+ audioStream: AudioStreamInfo | null;
+ format: VideoFormat;
+}
+
+const toSelectedStreams = (info: VideoInfo) => ({
+ videoStream: info.videoStreams[0] ?? null,
+ audioStream: info.audioStreams[0] ?? null,
+ format: info.format,
+});
+
+export const probeStub = Object.fromEntries(
+ Object.entries(videoInfoStub).map(([key, info]) => [key, toSelectedStreams(info)]),
+) as Record<keyof typeof videoInfoStub, SelectedStreams>;
diff --git a/server/test/mappers.ts b/server/test/mappers.ts
index 8cac6ce8fd..53d1d00a13 100644
--- a/server/test/mappers.ts
+++ b/server/test/mappers.ts
@@ -1,9 +1,10 @@
-import { Selectable, ShallowDehydrateObject } from 'kysely';
+import { NotNull, Selectable, ShallowDehydrateObject } from 'kysely';
import { MapAsset } from 'src/dtos/asset-response.dto';
import { AssetEditActionItem } from 'src/dtos/editing.dto';
import { ActivityTable } from 'src/schema/tables/activity.table';
import { AssetTable } from 'src/schema/tables/asset.table';
import { PartnerTable } from 'src/schema/tables/partner.table';
+import { AudioStreamInfo, VideoFormat, VideoStreamInfo } from 'src/types';
import { AlbumFactory } from 'test/factories/album.factory';
import { AssetFaceFactory } from 'test/factories/asset-face.factory';
import { AssetFactory } from 'test/factories/asset.factory';
@@ -155,6 +156,9 @@ export const getForGenerateThumbnail = (asset: ReturnType<AssetFactory['build']>
files: asset.files.map((file) => getDehydrated(file)),
exifInfo: getDehydrated(asset.exifInfo),
edits: asset.edits.map(({ action, parameters }) => ({ action, parameters })) as AssetEditActionItem[],
+ videoStream: null as (VideoStreamInfo & { timeBase: NotNull }) | null,
+ audioStream: null as AudioStreamInfo | null,
+ format: null as VideoFormat | null,
});
export const getForAssetFace = (face: ReturnType<AssetFaceFactory['build']>) => ({
diff --git a/server/test/medium.factory.ts b/server/test/medium.factory.ts
index c2019029da..8e3372011a 100644
--- a/server/test/medium.factory.ts
+++ b/server/test/medium.factory.ts
@@ -35,6 +35,7 @@ import { JobRepository } from 'src/repositories/job.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { MachineLearningRepository } from 'src/repositories/machine-learning.repository';
import { MapRepository } from 'src/repositories/map.repository';
+import { MediaRepository } from 'src/repositories/media.repository';
import { MemoryRepository } from 'src/repositories/memory.repository';
import { MetadataRepository } from 'src/repositories/metadata.repository';
import { NotificationRepository } from 'src/repositories/notification.repository';
@@ -218,7 +219,7 @@ export class MediumTestContext<S extends BaseService = BaseService> {
}
async newExif(dto: Insertable<AssetExifTable>) {
- const result = await this.get(AssetRepository).upsertExif(dto, { lockedPropertiesBehavior: 'override' });
+ const result = await this.get(AssetRepository).upsertExif({ exif: dto, lockedPropertiesBehavior: 'override' });
return { result };
}
@@ -362,7 +363,14 @@ export class ExifTestContext extends MediumTestContext<MetadataService> {
constructor(database: Kysely<DB>) {
super(MetadataService, {
database,
- real: [AssetRepository, AssetJobRepository, MetadataRepository, SystemMetadataRepository, TagRepository],
+ real: [
+ AssetRepository,
+ AssetJobRepository,
+ MediaRepository,
+ MetadataRepository,
+ SystemMetadataRepository,
+ TagRepository,
+ ],
mock: [ConfigRepository, EventRepository, LoggingRepository, MapRepository, StorageRepository],
});
@@ -445,6 +453,7 @@ const newRealRepository = <T>(key: ClassConstructor<T>, db: Kysely<DB>): T => {
return new key(LoggingRepository.create());
}
+ case MediaRepository:
case MetadataRepository: {
return new key(LoggingRepository.create());
}
diff --git a/server/test/medium/specs/exif/audio-video.spec.ts b/server/test/medium/specs/exif/audio-video.spec.ts
new file mode 100644
index 0000000000..430e7826f9
--- /dev/null
+++ b/server/test/medium/specs/exif/audio-video.spec.ts
@@ -0,0 +1,142 @@
+import { Kysely } from 'kysely';
+import { resolve } from 'node:path';
+import {
+ AacProfile,
+ AssetType,
+ ColorMatrix,
+ ColorPrimaries,
+ ColorTransfer,
+ DvProfile,
+ DvSignalCompatibility,
+ H264Profile,
+ HevcProfile,
+} from 'src/enum';
+import { DB } from 'src/schema';
+import { ExifTestContext, testAssetsDir } from 'test/medium.factory';
+import { getKyselyDB } from 'test/utils';
+
+let database: Kysely<DB>;
+
+beforeAll(async () => {
+ database = await getKyselyDB();
+});
+
+const fixtures = [
+ {
+ file: 'eiffel-tower.mp4',
+ video: {
+ codecName: 'h264',
+ formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
+ formatLongName: 'QuickTime / MOV',
+ pixelFormat: 'yuv420p',
+ bitrate: 5_128_622,
+ frameCount: 557,
+ timeBase: 90_000,
+ index: 0,
+ profile: H264Profile.High,
+ level: 40,
+ colorPrimaries: ColorPrimaries.Smpte170M,
+ colorTransfer: ColorTransfer.Smpte170M,
+ colorMatrix: ColorMatrix.Smpte170M,
+ dvProfile: null,
+ dvLevel: null,
+ dvBlSignalCompatibilityId: null,
+ },
+ audio: { codecName: 'aac', bitrate: 125_629, index: 1, profile: AacProfile.Lc },
+ keyframes: {
+ totalDuration: 2_012_441,
+ packetCount: 557,
+ outputFrames: 557,
+ pts: [0, 462_502, 925_004, 1_210_454, 1_387_506, 1_542_878, 1_850_008],
+ accDuration: [3613, 466_077, 928_541, 1_213_968, 1_391_005, 1_546_364, 1_853_469],
+ ownDuration: [3613, 3613, 3613, 3613, 3613, 3613, 3613],
+ },
+ },
+ {
+ file: 'waterfall.mp4',
+ video: {
+ codecName: 'hevc',
+ formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
+ formatLongName: 'QuickTime / MOV',
+ pixelFormat: 'yuvj420p',
+ bitrate: 43_363_499,
+ frameCount: 309,
+ timeBase: 90_000,
+ index: 2,
+ profile: HevcProfile.Main,
+ level: 156,
+ colorPrimaries: ColorPrimaries.Bt709,
+ colorTransfer: ColorTransfer.Bt709,
+ colorMatrix: ColorMatrix.Bt709,
+ dvProfile: null,
+ dvLevel: null,
+ dvBlSignalCompatibilityId: null,
+ },
+ audio: { codecName: 'aac', bitrate: 191_878, index: 1, profile: null },
+ keyframes: {
+ totalDuration: 932_286,
+ packetCount: 309,
+ outputFrames: 309,
+ pts: [0, 89_987, 179_974, 269_961, 359_948, 449_936, 539_923, 629_910, 725_166, 815_273, 905_295],
+ accDuration: [2999, 92_987, 182_974, 272_961, 362_948, 452_934, 542_922, 632_909, 728_175, 818_274, 908_296],
+ ownDuration: [2999, 3000, 3000, 3000, 3000, 2998, 2999, 2999, 3009, 3001, 3001],
+ },
+ },
+ {
+ file: 'train.mov',
+ video: {
+ codecName: 'hevc',
+ formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
+ formatLongName: 'QuickTime / MOV',
+ pixelFormat: 'yuv420p10le',
+ bitrate: 12_595_191,
+ frameCount: 1229,
+ timeBase: 600,
+ index: 0,
+ profile: HevcProfile.Main10,
+ level: 123,
+ colorPrimaries: ColorPrimaries.Bt2020,
+ colorTransfer: ColorTransfer.AribStdB67,
+ colorMatrix: ColorMatrix.Bt2020Nc,
+ dvProfile: DvProfile.Dvhe08,
+ dvLevel: 5,
+ dvBlSignalCompatibilityId: DvSignalCompatibility.Hlg,
+ },
+ audio: { codecName: 'aac', bitrate: 175_477, index: 1, profile: AacProfile.Lc },
+ keyframes: {
+ totalDuration: 12_290,
+ packetCount: 1229,
+ outputFrames: 1303,
+ pts: [
+ 0, 601, 1201, 1802, 2402, 3003, 3604, 4204, 4805, 5405, 6006, 6607, 7207, 7808, 8408, 9009, 9609, 10_210,
+ 10_811, 11_411, 12_062, 12_703,
+ ],
+ accDuration: [
+ 10, 580, 1180, 1780, 2380, 2980, 3580, 4180, 4780, 5380, 5980, 6580, 7180, 7780, 8380, 8980, 9580, 10_180,
+ 10_780, 11_380, 11_780, 12_100,
+ ],
+ ownDuration: [10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10],
+ },
+ },
+];
+
+const isExpected = <T extends keyof DB>(name: T, id: string, expected: Omit<DB[T], 'assetId'>) => {
+ const { table, ref } = database.dynamic;
+ const res = database.selectFrom(table(name).as('t')).selectAll().where(ref('assetId'), '=', id).executeTakeFirst();
+ return expect(res).resolves.toEqual({ ...expected, assetId: id });
+};
+
+describe('video metadata extraction', () => {
+ it.each(fixtures)('$file', async ({ file, video, audio, keyframes }) => {
+ const ctx = new ExifTestContext(database);
+ const { user } = await ctx.newUser();
+ const originalPath = resolve(testAssetsDir, 'videos', file);
+ const { asset } = await ctx.newAsset({ ownerId: user.id, originalPath, type: AssetType.Video });
+
+ await ctx.sut.handleMetadataExtraction({ id: asset.id });
+
+ await isExpected('asset_audio', asset.id, audio);
+ await isExpected('asset_video', asset.id, video);
+ await isExpected('asset_keyframe', asset.id, keyframes);
+ });
+});
diff --git a/server/test/medium/specs/repositories/asset.repository.spec.ts b/server/test/medium/specs/repositories/asset.repository.spec.ts
index 896489672e..2e449ae801 100644
--- a/server/test/medium/specs/repositories/asset.repository.spec.ts
+++ b/server/test/medium/specs/repositories/asset.repository.spec.ts
@@ -98,10 +98,10 @@ describe(AssetRepository.name, () => {
.executeTakeFirstOrThrow(),
).resolves.toEqual({ lockedProperties: ['dateTimeOriginal'] });
- await sut.upsertExif(
- { assetId: asset.id, lockedProperties: ['description'] },
- { lockedPropertiesBehavior: 'append' },
- );
+ await sut.upsertExif({
+ exif: { assetId: asset.id, lockedProperties: ['description'] },
+ lockedPropertiesBehavior: 'append',
+ });
await expect(
ctx.database
@@ -130,10 +130,10 @@ describe(AssetRepository.name, () => {
.executeTakeFirstOrThrow(),
).resolves.toEqual({ lockedProperties: ['dateTimeOriginal', 'description'] });
- await sut.upsertExif(
- { assetId: asset.id, lockedProperties: ['description'] },
- { lockedPropertiesBehavior: 'append' },
- );
+ await sut.upsertExif({
+ exif: { assetId: asset.id, lockedProperties: ['description'] },
+ lockedPropertiesBehavior: 'append',
+ });
await expect(
ctx.database
diff --git a/server/test/medium/specs/sync/sync-album-asset-exif.spec.ts b/server/test/medium/specs/sync/sync-album-asset-exif.spec.ts
index 1865fc2c80..8e1529edb0 100644
--- a/server/test/medium/specs/sync/sync-album-asset-exif.spec.ts
+++ b/server/test/medium/specs/sync/sync-album-asset-exif.spec.ts
@@ -289,13 +289,13 @@ describe(SyncRequestType.AlbumAssetExifsV1, () => {
// update the asset
const assetRepository = ctx.get(AssetRepository);
- await assetRepository.upsertExif(
- updateLockedColumns({
+ await assetRepository.upsertExif({
+ exif: updateLockedColumns({
assetId: asset.id,
city: 'New City',
}),
- { lockedPropertiesBehavior: 'append' },
- );
+ lockedPropertiesBehavior: 'append',
+ });
await expect(ctx.syncStream(auth, [SyncRequestType.AlbumAssetExifsV1])).resolves.toEqual([
{
@@ -350,13 +350,13 @@ describe(SyncRequestType.AlbumAssetExifsV1, () => {
// update the asset
const assetRepository = ctx.get(AssetRepository);
- await assetRepository.upsertExif(
- updateLockedColumns({
+ await assetRepository.upsertExif({
+ exif: updateLockedColumns({
assetId: assetDelayedExif.id,
city: 'Delayed Exif',
}),
- { lockedPropertiesBehavior: 'append' },
- );
+ lockedPropertiesBehavior: 'append',
+ });
await expect(ctx.syncStream(auth, [SyncRequestType.AlbumAssetExifsV1])).resolves.toEqual([
{
diff --git a/server/test/repositories/media.repository.mock.ts b/server/test/repositories/media.repository.mock.ts
index bd8deb4b3a..a1fc55cc3d 100644
--- a/server/test/repositories/media.repository.mock.ts
+++ b/server/test/repositories/media.repository.mock.ts
@@ -11,6 +11,14 @@ export const newMediaRepositoryMock = (): Mocked<RepositoryInterface<MediaReposi
decodeImage: vitest.fn().mockResolvedValue({ data: Buffer.from(''), info: {} }),
extract: vitest.fn().mockResolvedValue(null),
probe: vitest.fn(),
+ probePackets: vitest.fn().mockResolvedValue({
+ totalDuration: 0,
+ packetCount: 0,
+ outputFrames: 0,
+ keyframePts: [],
+ keyframeAccDuration: [],
+ keyframeOwnDuration: [],
+ }),
transcode: vitest.fn(),
getImageMetadata: vitest.fn(),
};