Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Reduce rows returned when listing comments

Chocobozzz38b16d6ed049
diff --git a/packages/tests/src/api/videos/video-comments.ts b/packages/tests/src/api/videos/video-comments.ts
index e33286f117..e805c6a1ea 100644
--- a/packages/tests/src/api/videos/video-comments.ts
+++ b/packages/tests/src/api/videos/video-comments.ts
@@ -524,6 +524,10 @@ describe('Test video comments', function () {
expect(threads.total).to.equal(0)
}
})
+
+ after(async function () {
+ await cleanupTests([ server2, server3 ])
+ })
})
after(async function () {
diff --git a/server/core/models/actor/actor-image.ts b/server/core/models/actor/actor-image.ts
index 4176bd0fbe..e70cc78855 100644
--- a/server/core/models/actor/actor-image.ts
+++ b/server/core/models/actor/actor-image.ts
@@ -80,6 +80,15 @@ export class ActorImageModel extends SequelizeModel<ActorImageModel> {
// ---------------------------------------------------------------------------
+ static getSQLAttributesJSON () {
+ return buildSQLAttributes({
+ model: this,
+ tableName: '',
+ aliasPrefix: '',
+ jsonMode: true
+ })
+ }
+
static getSQLAttributes (tableName: string, aliasPrefix = '') {
return buildSQLAttributes({
model: this,
diff --git a/server/core/models/shared/model-builder.ts b/server/core/models/shared/model-builder.ts
index 89b6348981..5428fa878e 100644
--- a/server/core/models/shared/model-builder.ts
+++ b/server/core/models/shared/model-builder.ts
@@ -1,3 +1,4 @@
+import { arrayify } from '@peertube/peertube-core-utils'
import { logger } from '@server/helpers/logger.js'
import isPlainObject from 'lodash-es/isPlainObject.js'
import { ModelStatic, Sequelize, Model as SequelizeModel } from 'sequelize'
@@ -64,13 +65,14 @@ export class ModelBuilder<T extends SequelizeModel> {
const { created, model } = this.createOrFindModel(json, rootTableName, keyPath)
for (const key of Object.keys(json)) {
- const value = json[key]
- if (!value) continue
+ const valueOrArray = json[key]
+ if (!valueOrArray) continue
const tableName = this.buildTableName(key)
- // Child model
- if (isPlainObject(value)) {
+ // Child model?
+ // Array of children or plain object
+ if ((Array.isArray(valueOrArray) && valueOrArray.length !== 0 && isPlainObject(valueOrArray[0])) || isPlainObject(valueOrArray)) {
const Model = this.findModelBuilder(rootTableName)
const association = Model.associations[tableName]
@@ -85,18 +87,22 @@ export class ModelBuilder<T extends SequelizeModel> {
continue
}
- const { created, model: subModel } = this.createModel(value, tableName, `${keyPath}.${json.id}.${key}`)
+ const valueArray = arrayify(valueOrArray)
- if (association.isMultiAssociation && !Array.isArray(model[tableName])) {
- model[tableName] = []
- }
+ for (const value of valueArray) {
+ const { created, model: subModel } = this.createModel(value, tableName, `${keyPath}.${json.id}.${key}`)
+
+ if (association.isMultiAssociation && !Array.isArray(model[tableName])) {
+ model[tableName] = []
+ }
- if (!created || !subModel) continue
+ if (!created || !subModel) continue
- if (Array.isArray(model[tableName])) {
- model[tableName].push(subModel)
- } else {
- model[tableName] = subModel
+ if (Array.isArray(model[tableName])) {
+ model[tableName].push(subModel)
+ } else {
+ model[tableName] = subModel
+ }
}
}
}
diff --git a/server/core/models/shared/sql/actor-helpers.ts b/server/core/models/shared/sql/actor-helpers.ts
index 600bb53c3b..53e825cf1d 100644
--- a/server/core/models/shared/sql/actor-helpers.ts
+++ b/server/core/models/shared/sql/actor-helpers.ts
@@ -99,3 +99,25 @@ export function getAvatarsJoin (options: {
`ON "${base}Avatars"."actorId" = ${on} ` +
`AND "${base}Avatars"."type" = ${ActorImageType.AVATAR} `
}
+
+/**
+ * Build JSON directly in PostgreSQL to reduce amount of rows transferred and processed in JS
+ */
+export function getAvatarsJSONJoin (options: {
+ attributes: string
+ base: string
+ on: string
+}) {
+ const { attributes, base, on } = options
+
+ return `LEFT JOIN LATERAL (` +
+ `SELECT json_agg(` +
+ ` jsonb_build_object(${attributes})` +
+ `) AS "Avatars"` +
+ ` FROM "actorImage" WHERE "actorId" = ${on} AND "type" = ${ActorImageType.AVATAR}` +
+ `) AS "${base}AvatarsJSON" ON TRUE `
+}
+
+export function getAvatarsJSONAttributes (base: string) {
+ return `"${base}AvatarsJSON"."Avatars" AS "${base.replace(/->/g, '.')}Avatars"`
+}
diff --git a/server/core/models/shared/table.ts b/server/core/models/shared/table.ts
index 63b3ffb623..bb81aa5023 100644
--- a/server/core/models/shared/table.ts
+++ b/server/core/models/shared/table.ts
@@ -5,6 +5,8 @@ export function buildSQLAttributes<M extends Model> (options: {
model: ModelStatic<M>
tableName: string
+ jsonMode?: boolean // default false
+
excludeAttributes?: readonly Exclude<keyof AttributesOnly<M>, symbol>[]
includeAttributes?: readonly Exclude<keyof AttributesOnly<M>, symbol>[]
@@ -12,7 +14,7 @@ export function buildSQLAttributes<M extends Model> (options: {
idBuilder?: string[]
}) {
- const { model, tableName, aliasPrefix = '', excludeAttributes, includeAttributes, idBuilder } = options
+ const { model, tableName, aliasPrefix = '', excludeAttributes, includeAttributes, idBuilder, jsonMode = false } = options
const attributes = Object.keys(model.getAttributes()) as Exclude<keyof AttributesOnly<M>, symbol>[]
@@ -30,6 +32,10 @@ export function buildSQLAttributes<M extends Model> (options: {
return false
})
.map(a => {
+ if (jsonMode) {
+ return `'${a}', "${a}"`
+ }
+
return `"${tableName}"."${a}" AS "${aliasPrefix}${a}"`
})
diff --git a/server/core/models/video/sql/comment/video-comment-list-query-builder.ts b/server/core/models/video/sql/comment/video-comment-list-query-builder.ts
index 6f249a8654..90608c62ed 100644
--- a/server/core/models/video/sql/comment/video-comment-list-query-builder.ts
+++ b/server/core/models/video/sql/comment/video-comment-list-query-builder.ts
@@ -1,6 +1,6 @@
import { VideoChannelCollaboratorState, VideoPrivacy } from '@peertube/peertube-models'
import { AbstractListQuery, AbstractListQueryOptions } from '@server/models/shared/abstract-list-query.js'
-import { getAccountJoin, getActorJoin, getAvatarsJoin, getChannelJoin } from '@server/models/shared/sql/actor-helpers.js'
+import { getAccountJoin, getActorJoin, getAvatarsJSONJoin, getChannelJoin } from '@server/models/shared/sql/actor-helpers.js'
import { Sequelize } from 'sequelize'
import { createSafeIn } from '../../../shared/index.js'
import { VideoCommentTableAttributes } from './video-comment-table-attributes.js'
@@ -326,7 +326,8 @@ export class VideoCommentListQueryBuilder extends AbstractListQuery {
private buildAccountAvatarsJoin () {
if (this.builtAccountAvatarJoin) return
- this.join += getAvatarsJoin({
+ this.join += getAvatarsJSONJoin({
+ attributes: this.tableAttributes.getAvatarAttributesJSON(),
base: 'Account->Actor->',
on: '"VideoCommentModel"."Account.Actor.id"'
})
@@ -337,7 +338,8 @@ export class VideoCommentListQueryBuilder extends AbstractListQuery {
private buildChannelAvatarsJoin () {
if (this.builtChannelAvatarJoin) return
- this.join += getAvatarsJoin({
+ this.join += getAvatarsJSONJoin({
+ attributes: this.tableAttributes.getAvatarAttributesJSON(),
base: 'Video->VideoChannel->Actor->',
on: '"VideoCommentModel"."Video.VideoChannel.Actor.id"'
})
diff --git a/server/core/models/video/sql/comment/video-comment-table-attributes.ts b/server/core/models/video/sql/comment/video-comment-table-attributes.ts
index 33df439569..7afcfc1c75 100644
--- a/server/core/models/video/sql/comment/video-comment-table-attributes.ts
+++ b/server/core/models/video/sql/comment/video-comment-table-attributes.ts
@@ -4,10 +4,11 @@ import { ActorImageModel } from '@server/models/actor/actor-image.js'
import { ActorModel } from '@server/models/actor/actor.js'
import { CommentAutomaticTagModel } from '@server/models/automatic-tag/comment-automatic-tag.js'
import { ServerModel } from '@server/models/server/server.js'
+import { getAvatarsJSONAttributes } from '@server/models/shared/sql/actor-helpers.js'
import { buildSQLAttributes } from '@server/models/shared/table.js'
import { AutomaticTagModel } from '../../../automatic-tag/automatic-tag.js'
-import { VideoCommentModel } from '../../video-comment.js'
import { VideoChannelModel } from '../../video-channel.js'
+import { VideoCommentModel } from '../../video-comment.js'
export class VideoCommentTableAttributes {
@Memoize()
@@ -26,6 +27,13 @@ export class VideoCommentTableAttributes {
// ---------------------------------------------------------------------------
+ @Memoize()
+ getAvatarAttributesJSON () {
+ return ActorImageModel.getSQLAttributesJSON().join(', ')
+ }
+
+ // ---------------------------------------------------------------------------
+
@Memoize()
getAccountAttributes () {
return AccountModel.getSQLAttributes('Account', 'Account.').join(', ')
@@ -43,7 +51,7 @@ export class VideoCommentTableAttributes {
@Memoize()
getAccountAvatarAttributes () {
- return ActorImageModel.getSQLAttributes('Account->Actor->Avatars', 'Account.Actor.Avatars.').join(', ')
+ return getAvatarsJSONAttributes('Account->Actor->')
}
// ---------------------------------------------------------------------------
@@ -65,7 +73,7 @@ export class VideoCommentTableAttributes {
@Memoize()
getChannelAvatarAttributes () {
- return ActorImageModel.getSQLAttributes('Video->VideoChannel->Actor->Avatars', 'Video.VideoChannel.Actor.Avatars.').join(', ')
+ return getAvatarsJSONAttributes('Video->VideoChannel->Actor->')
}
// ---------------------------------------------------------------------------