ActivityPub-federated video streaming platform using P2P directly in your web browser
Optimize videos list SQL query
Chocobozzz576b8f8ede55
diff --git a/server/core/initializers/database.ts b/server/core/initializers/database.tsindex f503472500..0b194c9ce9 100644--- a/server/core/initializers/database.ts+++ b/server/core/initializers/database.ts@@ -128,7 +128,7 @@ export const sequelizeTypescript = new SequelizeTypescript({newMessage += ' in ' + executionTimeMs + 'ms'}- logger.debug(newMessage, { sql: message, executionTimeMs, tags: [ 'sql' ] })+ logger.debug(newMessage, { sql: message, tags: [ 'sql' ] })}})diff --git a/server/core/models/video/sql/video/videos-id-list-query-builder.ts b/server/core/models/video/sql/video/videos-id-list-query-builder.tsindex 767141faeb..3da5be68a8 100644--- a/server/core/models/video/sql/video/videos-id-list-query-builder.ts+++ b/server/core/models/video/sql/video/videos-id-list-query-builder.ts@@ -101,6 +101,8 @@ export type BuildVideosListQueryOptions = {excludeAlreadyWatched?: boolean}+type SortDirection = 'ASC' | 'DESC'+export class VideosIdListQueryBuilder extends AbstractRunQuery {protected replacements: any = {}@@ -115,6 +117,7 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {private having = ''private sort = ''+private limit = ''private offset = ''@@ -153,6 +156,16 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {if (options.group) this.group = options.groupif (options.having) this.having = options.having+ let sortColumn: string+ let sortDirection: SortDirection++ if (options.sort) {+ const { direction, field } = buildSortDirectionAndField(options.sort)++ sortColumn = field+ sortDirection = direction+ }+this.joins = this.joins.concat(['INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId"','INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId"',@@ -199,7 +212,7 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {}if (options.displayOnlyForFollower) {- this.whereFollowerActorId({ ...options.displayOnlyForFollower, isCount: options.isCount === true })+ this.whereFollowerActorId({ ...options.displayOnlyForFollower, isCount: options.isCount === true, sortColumn })}if (options.hasFiles === true) {@@ -265,10 +278,10 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {// We don't exclude results in this so if we do a count we don't need to add this complex clauseif (options.isCount !== true) {- if (options.sort.endsWith('trending')) {+ if (sortColumn === 'trending') {this.groupForTrending(options.trendingDays)- } else if (options.sort.endsWith('hot') || options.sort.endsWith('best')) {- this.addAttributeForHotOrBest(options.sort, options.user)+ } else if (sortColumn === 'hot' || sortColumn === 'best') {+ this.addAttributeForHotOrBest(sortColumn, options.user)}}@@ -313,8 +326,8 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {if (options.isCount === true) {this.setCountAttribute()} else {- if (exists(options.sort)) {- this.setSort(options.sort)+ if (exists(sortColumn)) {+ this.setSort(sortColumn, sortDirection)}if (exists(options.count)) {@@ -457,9 +470,19 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {this.replacements.channelOneOf = channelOneOf}- private whereFollowerActorId (options: { actorId: number, orLocalVideos: boolean, isCount: boolean }) {- // EXISTS doesn't work very well with COUNT queries, so we use a CTE instead- if (options.isCount) {+ private whereFollowerActorId (options: { sortColumn: string, actorId: number, orLocalVideos: boolean, isCount: boolean }) {+ const complexSorts = new Set([+ 'trending',+ 'hot',+ 'best',+ 'match',+ 'localVideoFilesSize'+ ])+ const isComplexSort = options.sortColumn && complexSorts.has(options.sortColumn)++ // EXISTS doesn't work very well with COUNT queries or complex sorts where PostgreSQL has to compute follow constraint for many rows+ // So we use a CTE instead+ if (options.isCount || isComplexSort) {// Don't use CTE on purpose, that seems to be slower in this caseconst targetActorIdQuery =`SELECT "targetActorId" FROM "actorFollow" WHERE "actorFollow"."actorId" = :followerActorId AND "actorFollow"."state" = 'accepted'`@@ -836,12 +859,12 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {this.attributes.push(attribute)}- private setSort (sort: string) {- if (sort === '-originallyPublishedAt' || sort === 'originallyPublishedAt') {+ private setSort (column: string, direction: 'ASC' | 'DESC') {+ if (column === 'originallyPublishedAt') {this.attributes.push('COALESCE("video"."originallyPublishedAt", "video"."publishedAt") AS "publishedAtForOrder"')}- if (sort === '-localVideoFilesSize' || sort === 'localVideoFilesSize') {+ if (column === 'localVideoFilesSize') {this.attributes.push('(' +'CASE ' +@@ -864,32 +887,31 @@ export class VideosIdListQueryBuilder extends AbstractRunQuery {)}- this.sort = this.buildOrder(sort)+ this.sort = this.buildOrder(column, direction)}- private buildOrder (value: string) {- const { direction, field } = buildSortDirectionAndField(value)- if (field.match(/^[a-zA-Z."]+$/) === null) throw new Error('Invalid sort column ' + field)+ private buildOrder (column: string, direction: 'ASC' | 'DESC') {+ if (column.match(/^[a-zA-Z."]+$/) === null) throw new Error('Invalid sort column ' + column)- if (field.toLowerCase() === 'random') return 'ORDER BY RANDOM()'- if (field.toLowerCase() === 'total') return `ORDER BY "total" ${direction}`+ if (column === 'random') return 'ORDER BY RANDOM()'+ if (column === 'total') return `ORDER BY "total" ${direction}`- if ([ 'trending', 'hot', 'best' ].includes(field.toLowerCase())) { // Sort by aggregation+ if ([ 'trending', 'hot', 'best' ].includes(column)) { // Sort by aggregationreturn `ORDER BY "score" ${direction}, "video"."views" ${direction}`}let firstSort: string- if (field.toLowerCase() === 'match') { // Search+ if (column === 'match') { // SearchfirstSort = '"similarity"'- } else if (field === 'originallyPublishedAt') {+ } else if (column === 'originallyPublishedAt') {firstSort = '"publishedAtForOrder"'- } else if (field === 'localVideoFilesSize') {+ } else if (column === 'localVideoFilesSize') {firstSort = '"localVideoFilesSize"'- } else if (field.includes('.')) {- firstSort = field+ } else if (column.includes('.')) {+ firstSort = column} else {- firstSort = `"video"."${field}"`+ firstSort = `"video"."${column}"`}return `ORDER BY ${firstSort} ${direction}, "video"."id" ASC`