ActivityPub-federated video streaming platform using P2P directly in your web browser
Optimize listing video redundancies
Chocobozzz8126d82cf2c9
diff --git a/server/core/lib/files-cache/shared/abstract-file-cache.ts b/server/core/lib/files-cache/shared/abstract-file-cache.tsindex 435bc1905e..d978fdc17c 100644--- a/server/core/lib/files-cache/shared/abstract-file-cache.ts+++ b/server/core/lib/files-cache/shared/abstract-file-cache.ts@@ -35,7 +35,7 @@ export abstract class AbstractFileCache<M extends FileModel> {const { filename, res, next } = optionsif (this.filenameToPathCache.has(filename)) {- return res.sendFile(this.filenameToPathCache.get(filename), { maxAge: STATIC_MAX_AGE.SERVER })+ return res.sendFile(this.filenameToPathCache.get(filename), { maxAge: STATIC_MAX_AGE.LAZY_SERVER })}const file = await this.lazyLoadIfNeeded(filename)diff --git a/server/core/models/redundancy/video-redundancy.ts b/server/core/models/redundancy/video-redundancy.tsindex f41c3698a0..94938b15ca 100644--- a/server/core/models/redundancy/video-redundancy.ts+++ b/server/core/models/redundancy/video-redundancy.ts@@ -1,6 +1,7 @@import {CacheFileObject,RedundancyInformation,+ VideoInclude,VideoPrivacy,VideoRedundanciesTarget,VideoRedundancy,@@ -11,7 +12,7 @@ import { isTestInstance } from '@peertube/peertube-node-utils'import { getServerActor } from '@server/models/application/application.js'import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models/index.js'import sample from 'lodash-es/sample.js'-import { literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize'+import { literal, Op, QueryTypes, Transaction } from 'sequelize'import {AllowNull,BeforeDestroy,@@ -31,12 +32,12 @@ import { CONFIG } from '../../initializers/config.js'import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js'import { ActorModel } from '../actor/actor.js'import { ServerModel } from '../server/server.js'-import { getSort, getVideoSort, parseAggregateResult, SequelizeModel, throwIfNotValid } from '../shared/index.js'+import { getVideoSort, parseAggregateResult, SequelizeModel, throwIfNotValid } from '../shared/index.js'import { ScheduleVideoUpdateModel } from '../video/schedule-video-update.js'import { VideoChannelModel } from '../video/video-channel.js'-import { VideoFileModel } from '../video/video-file.js'import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist.js'import { VideoModel } from '../video/video.js'+import { getAllPrivacies } from '@peertube/peertube-core-utils'export enum ScopeNames {WITH_VIDEO = 'WITH_VIDEO'@@ -443,7 +444,7 @@ export class VideoRedundancyModel extends SequelizeModel<VideoRedundancyModel> {return VideoRedundancyModel.findAll(query)}- static listForApi (options: {+ static async listForApi (options: {start: numbercount: numbersort: string@@ -451,68 +452,32 @@ export class VideoRedundancyModel extends SequelizeModel<VideoRedundancyModel> {strategy?: string}) {const { start, count, sort, target, strategy } = options- const redundancyWhere: WhereOptions = {}- const videosWhere: WhereOptions = {}- let countRedundancyJoinWhere = ''-- if (target === 'my-videos') {- Object.assign(videosWhere, { remote: false })- } else if (target === 'remote-videos') {- Object.assign(videosWhere, { remote: true })- Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })-- countRedundancyJoinWhere = ' AND "videoRedundancy"."strategy" IS NOT NULL'- }-- if (strategy) {- Object.assign(redundancyWhere, { strategy })- }-- // /!\ On video model /!\- const findOptions = {- offset: start,- limit: count,- order: getSort(sort),- where: videosWhere,- include: [- {- required: true,- model: VideoStreamingPlaylistModel.unscoped(),- include: [- {- model: VideoRedundancyModel.unscoped(),- required: true,- where: redundancyWhere- },- {- model: VideoFileModel,- required: true- }- ]- }- ]- }-- return Promise.all([- VideoModel.findAll(findOptions),-- VideoModel.count({- where: {- ...videosWhere,-- id: {- [Op.in]: literal(- '(' +- 'SELECT "videoId" FROM "videoStreamingPlaylist" ' +- 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id ' +- countRedundancyJoinWhere +- ')'- )- }- }- })- ]).then(([ data, total ]) => ({ total, data }))+ return VideoModel.listForApi({+ start,+ count,+ sort,+ nsfw: null,+ displayOnlyForFollower: null,++ include: VideoInclude.FILES | VideoInclude.BLACKLISTED | VideoInclude.NOT_PUBLISHED_STATE | VideoInclude.BLOCKED_OWNER,+ privacyOneOf: getAllPrivacies(),+ skipPrivateIncludeCheck: true,++ isLocal: target === 'my-videos'+ ? true+ : false,++ redundancyStrategy: strategy,+ localRedundancy: target === 'remote-videos'+ ? true+ : undefined,+ includeRedundancy: true,+ hasRedundancy: true,+ tableAttributes: {+ fullRedundancy: true+ }+ })}static async getStats (strategy: VideoRedundancyStrategyWithManual) {diff --git a/server/core/models/video/sql/video/shared/abstract-video-query-builder.ts b/server/core/models/video/sql/video/shared/abstract-video-query-builder.tsindex a28cd042a5..8fe4c46310 100644--- a/server/core/models/video/sql/video/shared/abstract-video-query-builder.ts+++ b/server/core/models/video/sql/video/shared/abstract-video-query-builder.ts@@ -5,6 +5,7 @@ import validator from 'validator'import { AbstractRunQuery } from '../../../../shared/abstract-run-query.js'import { createSafeIn } from '../../../../shared/index.js'import { VideoTableAttributes } from './video-table-attributes.js'+import { TableAttributeOptions } from './table-attributes-options.model.js'/*** Abstract builder to create SQL query and fetch video models@@ -310,7 +311,7 @@ export class AbstractVideoQueryBuilder extends AbstractRunQuery {}}- protected includeStreamingPlaylistRedundancies () {+ protected includeStreamingPlaylistRedundancies (tableAttributeOptions: TableAttributeOptions) {this.addJoin('LEFT OUTER JOIN "videoRedundancy" AS "VideoStreamingPlaylists->RedundancyVideos" ' +'ON "VideoStreamingPlaylists"."id" = "VideoStreamingPlaylists->RedundancyVideos"."videoStreamingPlaylistId"'@@ -319,7 +320,7 @@ export class AbstractVideoQueryBuilder extends AbstractRunQuery {this.attributes = {...this.attributes,- ...this.buildAttributesObject('VideoStreamingPlaylists->RedundancyVideos', this.tables.getRedundancyAttributes())+ ...this.buildAttributesObject('VideoStreamingPlaylists->RedundancyVideos', this.tables.getRedundancyAttributes(tableAttributeOptions))}}diff --git a/server/core/models/video/sql/video/shared/table-attributes-options.model.ts b/server/core/models/video/sql/video/shared/table-attributes-options.model.tsnew file mode 100644index 0000000000..b067be9bed--- /dev/null+++ b/server/core/models/video/sql/video/shared/table-attributes-options.model.ts@@ -0,0 +1,3 @@+export interface TableAttributeOptions {+ fullRedundancy?: boolean+}diff --git a/server/core/models/video/sql/video/shared/video-file-query-builder.ts b/server/core/models/video/sql/video/shared/video-file-query-builder.tsindex 2d061be4a0..09dbf46587 100644--- a/server/core/models/video/sql/video/shared/video-file-query-builder.ts+++ b/server/core/models/video/sql/video/shared/video-file-query-builder.ts@@ -1,5 +1,6 @@import { Sequelize, Transaction } from 'sequelize'import { AbstractVideoQueryBuilder } from './abstract-video-query-builder.js'+import { TableAttributeOptions } from './table-attributes-options.model.js'export type FileQueryOptions = {id?: string | number@@ -10,6 +11,8 @@ export type FileQueryOptions = {transaction?: Transactionlogging?: boolean++ tableAttributes?: TableAttributeOptions}/**@@ -53,7 +56,7 @@ export class VideoFileQueryBuilder extends AbstractVideoQueryBuilder {this.includeStreamingPlaylistFiles()if (options.includeRedundancy) {- this.includeStreamingPlaylistRedundancies()+ this.includeStreamingPlaylistRedundancies(options.tableAttributes)}this.whereId(options)diff --git a/server/core/models/video/sql/video/shared/video-model-builder.ts b/server/core/models/video/sql/video/shared/video-model-builder.tsindex 4bddcf2362..4837ca2d00 100644--- a/server/core/models/video/sql/video/shared/video-model-builder.ts+++ b/server/core/models/video/sql/video/shared/video-model-builder.ts@@ -23,6 +23,7 @@ import { VideoLiveModel } from '../../../video-live.js'import { VideoStreamingPlaylistModel } from '../../../video-streaming-playlist.js'import { VideoModel } from '../../../video.js'import { VideoTableAttributes } from './video-table-attributes.js'+import { TableAttributeOptions } from './table-attributes-options.model.js'type SQLRow = { [id: string]: string | number }@@ -54,6 +55,8 @@ export class VideoModelBuilder {private videos: VideoModel[]+ private tableAttributeOptions: TableAttributeOptions+private readonly buildOpts = { raw: true, isNewRecord: false }constructor (@@ -68,8 +71,12 @@ export class VideoModelBuilder {include?: VideoIncludeTyperowsWebVideoFiles?: SQLRow[]rowsStreamingPlaylist?: SQLRow[]++ tableAttributes?: TableAttributeOptions}) {- const { rows, rowsWebVideoFiles, rowsStreamingPlaylist, include, addCaptions } = options+ const { rows, rowsWebVideoFiles, rowsStreamingPlaylist, include, addCaptions, tableAttributes } = options++ this.tableAttributeOptions = tableAttributesthis.reinit()@@ -168,7 +175,7 @@ export class VideoModelBuilder {this.videos = []}- private grabSeparateWebVideoFiles (rowsWebVideoFiles?: SQLRow[]) {+ private grabSeparateWebVideoFiles (rowsWebVideoFiles: SQLRow[]) {if (!rowsWebVideoFiles) returnfor (const row of rowsWebVideoFiles) {@@ -180,7 +187,7 @@ export class VideoModelBuilder {}}- private grabSeparateStreamingPlaylistFiles (rowsStreamingPlaylist?: SQLRow[]) {+ private grabSeparateStreamingPlaylistFiles (rowsStreamingPlaylist: SQLRow[]) {if (!rowsStreamingPlaylist) returnfor (const row of rowsStreamingPlaylist) {@@ -359,7 +366,7 @@ export class VideoModelBuilder {if (!id || this.redundancyDone.has(id)) return- const attributes = this.grab(row, this.tables.getRedundancyAttributes(), redundancyPrefix)+ const attributes = this.grab(row, this.tables.getRedundancyAttributes(this.tableAttributeOptions), redundancyPrefix)const redundancyModel = new VideoRedundancyModel(attributes, this.buildOpts)to.RedundancyVideos.push(redundancyModel)diff --git a/server/core/models/video/sql/video/shared/video-table-attributes.ts b/server/core/models/video/sql/video/shared/video-table-attributes.tsindex 8551c4397b..37b654fdbd 100644--- a/server/core/models/video/sql/video/shared/video-table-attributes.ts+++ b/server/core/models/video/sql/video/shared/video-table-attributes.ts@@ -1,4 +1,5 @@import { thumbnailAPIAttributes } from '@server/models/video/thumbnail.js'+import { TableAttributeOptions } from './table-attributes-options.model.js'/*** Class to build video attributes/join names we want to fetch from the database@@ -212,7 +213,11 @@ export class VideoTableAttributes {return [ 'id', 'name' ]}- getRedundancyAttributes () {+ getRedundancyAttributes (tableAttributeOptions: TableAttributeOptions) {+ if (tableAttributeOptions?.fullRedundancy === true) {+ return [ 'id', 'strategy', 'createdAt', 'updatedAt', 'expiresOn', 'fileUrl' ]+ }+return [ 'id', 'fileUrl' ]}diff --git a/server/core/models/video/sql/video/video-model-get-query-builder.ts b/server/core/models/video/sql/video/video-model-get-query-builder.tsindex 9e642e4cb8..7d5ec7d5a0 100644--- a/server/core/models/video/sql/video/video-model-get-query-builder.ts+++ b/server/core/models/video/sql/video/video-model-get-query-builder.ts@@ -4,6 +4,7 @@ import { AbstractVideoQueryBuilder } from './shared/abstract-video-query-builderimport { VideoFileQueryBuilder } from './shared/video-file-query-builder.js'import { VideoModelBuilder } from './shared/video-model-builder.js'import { VideoTableAttributes } from './shared/video-table-attributes.js'+import { TableAttributeOptions } from './shared/table-attributes-options.model.js'/*** Build a GET SQL query, fetch rows and create the video model@@ -60,6 +61,8 @@ export type BuildVideoGetQueryOptions = {transaction?: Transactionlogging?: boolean++ tableAttributes?: TableAttributeOptions}export class VideoModelGetQueryBuilder {@@ -79,7 +82,7 @@ export class VideoModelGetQueryBuilder {async queryVideo (options: BuildVideoGetQueryOptions) {const fileQueryOptions = {- ...pick(options, [ 'id', 'url', 'transaction', 'logging' ]),+ ...pick(options, [ 'id', 'url', 'transaction', 'logging', 'tableAttributes' ]),includeRedundancy: this.shouldIncludeRedundancies(options)}diff --git a/server/core/models/video/sql/video/videos-id-list-query-builder.ts b/server/core/models/video/sql/video/videos-id-list-query-builder.tsindex 4ae4fe15a8..11737f8fd6 100644--- a/server/core/models/video/sql/video/videos-id-list-query-builder.ts+++ b/server/core/models/video/sql/video/videos-id-list-query-builder.ts@@ -105,6 +105,11 @@ export type BuildVideosListQueryOptions = {logging?: booleanexcludeAlreadyWatched?: boolean++ hasRedundancy?: boolean+ redundancyStrategy?: string+ includeRedundancy?: boolean+ localRedundancy?: boolean}type SortDirection = 'ASC' | 'DESC'@@ -366,6 +371,10 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {}}+ if (options.hasRedundancy === true) {+ this.whereRedundancyExists({ redundancyStrategy: options.redundancyStrategy, localRedundancy: options.localRedundancy })+ }+this.whereSearch(options)const cteString = this.cte.length !== 0@@ -458,6 +467,28 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {this.joins.push('INNER JOIN "actor" "channelActor" ON "videoChannel"."id" = "channelActor"."videoChannelId"')}+ private whereRedundancyExists (options: {+ redundancyStrategy?: string+ localRedundancy?: boolean+ }) {+ let strategySql = ''++ if (options.redundancyStrategy) {+ strategySql = ' AND "videoRedundancy"."strategy" = :redundancyStrategy'+ this.replacements.redundancyStrategy = options.redundancyStrategy+ } else if (options.localRedundancy === true) {+ strategySql = ' AND "videoRedundancy"."strategy" IS NOT NULL'+ }++ this.and.push(+ 'EXISTS (' ++ 'SELECT 1 FROM "videoStreamingPlaylist" ' ++ 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ' ++ 'WHERE "videoStreamingPlaylist"."videoId" = "video"."id"' + strategySql ++ ')'+ )+ }+private whereStateAvailable (options: {includeScheduledLive: boolean}) {diff --git a/server/core/models/video/sql/video/videos-model-list-query-builder.ts b/server/core/models/video/sql/video/videos-model-list-query-builder.tsindex 539a78d80c..259c6e6c84 100644--- a/server/core/models/video/sql/video/videos-model-list-query-builder.ts+++ b/server/core/models/video/sql/video/videos-model-list-query-builder.ts@@ -4,6 +4,7 @@ import { getServerActor } from '@server/models/application/application.js'import { MActorAccount } from '@server/types/models/index.js'import { Sequelize } from 'sequelize'import { AbstractVideoQueryBuilder } from './shared/abstract-video-query-builder.js'+import { TableAttributeOptions } from './shared/table-attributes-options.model.js'import { VideoFileQueryBuilder } from './shared/video-file-query-builder.js'import { VideoModelBuilder } from './shared/video-model-builder.js'import { BuildVideosListQueryOptions, VideosIdListQueryBuilder } from './videos-id-list-query-builder.js'@@ -12,6 +13,10 @@ import { BuildVideosListQueryOptions, VideosIdListQueryBuilder } from './videos-* Build videos list SQL query and create video models*/+export type QueryVideosListOptions = BuildVideosListQueryOptions & {+ tableAttributes?: TableAttributeOptions+}+export class VideosModelListQueryBuilder extends AbstractVideoQueryBuilder {private innerQuery: stringprivate innerSort: string@@ -29,7 +34,7 @@ export class VideosModelListQueryBuilder extends AbstractVideoQueryBuilder {this.streamingPlaylistFilesQueryBuilder = new VideoFileQueryBuilder(sequelize)}- async queryVideos (options: BuildVideosListQueryOptions) {+ async queryVideos (options: QueryVideosListOptions) {const serverActor = await getServerActor()this.buildInnerQuery(options)@@ -42,10 +47,10 @@ export class VideosModelListQueryBuilder extends AbstractVideoQueryBuilder {if (videoIds.length !== 0) {const fileQueryOptions = {- ...pick(options, [ 'transaction', 'logging' ]),+ ...pick(options, [ 'transaction', 'logging', 'tableAttributes' ]),ids: videoIds,- includeRedundancy: false+ includeRedundancy: options.includeRedundancy === true}const [ rowsWebVideoFiles, rowsStreamingPlaylist ] = await Promise.all([@@ -56,6 +61,7 @@ export class VideosModelListQueryBuilder extends AbstractVideoQueryBuilder {return this.videoModelBuilder.buildVideosFromRows({rows,include: options.include,+ tableAttributes: options.tableAttributes,addCaptions: false,rowsStreamingPlaylist,rowsWebVideoFilesdiff --git a/server/core/models/video/video.ts b/server/core/models/video/video.tsindex def8bbfd9d..65a4a1ea6d 100644--- a/server/core/models/video/video.ts+++ b/server/core/models/video/video.ts@@ -94,7 +94,6 @@ import {MStreamingPlaylist,MStreamingPlaylistFilesVideo,MUserAccountId,- MUserId,MVideoAP,MVideoAPLight,MVideoAccountLightBlacklistAllFiles,@@ -153,6 +152,7 @@ import { ScheduleVideoUpdateModel } from './schedule-video-update.js'import {BuildVideosListQueryOptions,DisplayOnlyForFollowerOptions,+ QueryVideosListOptions,VideoModelGetQueryBuilder,VideosIdListQueryBuilder,VideosModelListQueryBuilder@@ -1109,63 +1109,60 @@ export class VideoModel extends SequelizeModel<VideoModel> {return result.map(v => v.uuid)}- static async listForApi (options: {- start: number- count: number- sort: string-- nsfw: boolean- nsfwFlagsIncluded?: number- nsfwFlagsExcluded?: number-- isLive?: boolean- isLocal?: boolean- include?: VideoIncludeType- includeScheduledLive?: boolean-- hasFiles?: boolean // default false+ static async listForApi (+ options:+ & {+ start: number+ count: number+ sort: string- hasWebVideoFiles?: boolean-- hasHLSFiles?: boolean-- categoryOneOf?: number[]- licenceOneOf?: number[]- languageOneOf?: string[]- tagsOneOf?: string[]- tagsAllOf?: string[]- privacyOneOf?: VideoPrivacyType[]+ nsfw: boolean- host?: string+ countVideos?: boolean- accountId?: number-- videoChannelId?: number- channelNameOneOf?: string[]-- displayOnlyForFollower: DisplayOnlyForFollowerOptions | null-- videoPlaylistId?: number-- trendingDays?: number-- user?: MUserAccountId- historyOfUser?: MUserId-- countVideos?: boolean // default true-- search?: string-- excludeAlreadyWatched?: boolean // default false-- autoTagOneOf?: string[]-- stateOneOf?: VideoStateType[]-- includeCollaborations?: boolean // default false- }) {- VideoModel.throwIfPrivateIncludeWithoutUser(options)- VideoModel.throwIfPrivacyOneOfWithoutUser(options)+ skipPrivateIncludeCheck?: boolean+ }+ & Pick<+ QueryVideosListOptions,+ | 'nsfwFlagsIncluded'+ | 'nsfwFlagsExcluded'+ | 'isLive'+ | 'categoryOneOf'+ | 'licenceOneOf'+ | 'languageOneOf'+ | 'host'+ | 'autoTagOneOf'+ | 'stateOneOf'+ | 'tagsOneOf'+ | 'tagsAllOf'+ | 'privacyOneOf'+ | 'isLocal'+ | 'include'+ | 'includeScheduledLive'+ | 'displayOnlyForFollower'+ | 'hasFiles'+ | 'accountId'+ | 'includeCollaborations'+ | 'videoChannelId'+ | 'channelNameOneOf'+ | 'videoPlaylistId'+ | 'user'+ | 'historyOfUser'+ | 'hasHLSFiles'+ | 'hasWebVideoFiles'+ | 'search'+ | 'excludeAlreadyWatched'+ | 'hasRedundancy'+ | 'redundancyStrategy'+ | 'includeRedundancy'+ | 'localRedundancy'+ | 'tableAttributes'+ >+ ) {+ if (options.skipPrivateIncludeCheck !== true) {+ VideoModel.throwIfPrivateIncludeWithoutUser(options)+ VideoModel.throwIfPrivacyOneOfWithoutUser(options)+ }const serverActor = await getServerActor()@@ -1202,7 +1199,12 @@ export class VideoModel extends SequelizeModel<VideoModel> {'hasHLSFiles','hasWebVideoFiles','search',- 'excludeAlreadyWatched'+ 'excludeAlreadyWatched',+ 'redundancyStrategy',+ 'hasRedundancy',+ 'localRedundancy',+ 'includeRedundancy',+ 'tableAttributes']),serverAccountIdForBlock: serverActor.Account.id,@@ -1770,7 +1772,7 @@ export class VideoModel extends SequelizeModel<VideoModel> {// ---------------------------------------------------------------------------private static async getAvailableForApi (- options: BuildVideosListQueryOptions,+ options: QueryVideosListOptions,countVideos = true): Promise<ResultList<VideoModel>> {const span = tracer.startSpan('peertube.VideoModel.getAvailableForApi')@@ -1811,7 +1813,7 @@ export class VideoModel extends SequelizeModel<VideoModel> {if (!VideoModel.isPrivateInclude(include)) returnif (user?.hasRight(UserRight.SEE_ALL_VIDEOS)) return- if (user.Account.id === accountId) return+ if (accountId && user?.Account.id === accountId) returnthrow new Error('Try to include protected videos but user cannot see all videos')}@@ -1825,7 +1827,7 @@ export class VideoModel extends SequelizeModel<VideoModel> {if (!privacyOneOf) returnif (user?.hasRight(UserRight.SEE_ALL_VIDEOS)) return- if (user.Account.id === accountId) return+ if (accountId && user?.Account.id === accountId) returnthrow new Error('Try to choose video privacies but user cannot see all videos')}