Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Keep videos order on user import/export

Chocobozzz3c316a77362c
diff --git a/packages/tests/src/api/users/user-import.ts b/packages/tests/src/api/users/user-import.ts
index 6bdf125d04..cacab65b89 100644
--- a/packages/tests/src/api/users/user-import.ts
+++ b/packages/tests/src/api/users/user-import.ts
@@ -41,6 +41,8 @@ function runTest (withObjectStorage: boolean) {
let remoteNoahToken: string
let remoteNoahId: number
+ let noahVODNames: string[]
+
let archivePath: string
let objectStorage: ObjectStorageCommand
@@ -63,7 +65,8 @@ function runTest (withObjectStorage: boolean) {
remoteNoahToken,
remoteServer,
mouskaVideo,
- blockedServer
+ blockedServer,
+ noahVODNames
} = await prepareImportExportTests({ emails, objectStorage, withBlockedServer: true }))
await blockedServer.videos.quickUpload({ name: 'blocked video' })
@@ -92,6 +95,7 @@ function runTest (withObjectStorage: boolean) {
videoPasswords: [ 'password1', 'password2' ]
}
})
+ noahVODNames.unshift('noah password video')
// Add a video in watch later playlist
await server.playlists.addElement({
@@ -373,9 +377,13 @@ function runTest (withObjectStorage: boolean) {
// We observe weird behaviour in the CI, so re-wait jobs here just to be sure
await waitJobs([ remoteServer, server ])
- const { data } = await remoteServer.videos.listMyVideos({ token: remoteNoahToken })
+ const { data } = await remoteServer.videos.listMyVideos({ token: remoteNoahToken, sort: '-publishedAt' })
expect(data).to.have.lengthOf(5)
+ // Correct order
+ const vodNames = data.filter(v => !v.isLive).map(v => v.name)
+ expect(vodNames).to.deep.equal(noahVODNames)
+
{
const privateVideo = data.find(v => v.name === 'noah private video')
expect(privateVideo).to.exist
diff --git a/packages/tests/src/shared/import-export.ts b/packages/tests/src/shared/import-export.ts
index 7e98721513..1511277455 100644
--- a/packages/tests/src/shared/import-export.ts
+++ b/packages/tests/src/shared/import-export.ts
@@ -374,6 +374,11 @@ export async function prepareImportExportTests (options: {
token: noahToken
})
+ await waitJobs([ server, remoteServer ])
+
+ const { data: noahVideos } = await server.videos.listMyVideos({ token: noahToken, sort: '-publishedAt' })
+ const noahVODNames = noahVideos.filter(v => !v.isLive).map(v => v.name)
+
return {
rootId,
@@ -395,6 +400,8 @@ export async function prepareImportExportTests (options: {
server,
remoteServer,
- blockedServer
+ blockedServer,
+
+ noahVODNames
}
}
diff --git a/server/core/lib/user-import-export/exporters/videos-exporter.ts b/server/core/lib/user-import-export/exporters/videos-exporter.ts
index f1b8b4b1bb..f3f8050edf 100644
--- a/server/core/lib/user-import-export/exporters/videos-exporter.ts
+++ b/server/core/lib/user-import-export/exporters/videos-exporter.ts
@@ -1,7 +1,6 @@
-import { pick } from '@peertube/peertube-core-utils'
+import { pick, sortBy } from '@peertube/peertube-core-utils'
import { ActivityCreate, FileStorage, VideoCommentPolicy, VideoExportJSON, VideoObject, VideoPrivacy } from '@peertube/peertube-models'
import { logger } from '@server/helpers/logger.js'
-import { USER_EXPORT_MAX_ITEMS } from '@server/initializers/constants.js'
import { audiencify, getVideoAudience } from '@server/lib/activitypub/audience.js'
import { buildCreateActivity } from '@server/lib/activitypub/send/send-create.js'
import { buildChaptersAPHasPart } from '@server/lib/activitypub/video-chapters.js'
@@ -15,7 +14,6 @@ import { VideoDownload } from '@server/lib/video-download.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { PlayerSettingModel } from '@server/models/video/player-setting.js'
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
-import { VideoChannelModel } from '@server/models/video/video-channel.js'
import { VideoChapterModel } from '@server/models/video/video-chapter.js'
import { VideoLiveModel } from '@server/models/video/video-live.js'
import { VideoPasswordModel } from '@server/models/video/video-password.js'
@@ -56,10 +54,13 @@ export class VideosExporter extends AbstractUserExporter<VideoExportJSON> {
const activityPubOutbox: ActivityCreate<VideoObject>[] = []
let staticFiles: ExportResult<VideoExportJSON>['staticFiles'] = []
- const channels = await VideoChannelModel.listAllOwnedByAccount(this.user.Account.id)
+ let videoIds: number[] = []
+ let start = 0
+ const chunkSize = 100
- for (const channel of channels) {
- const videoIds = await VideoModel.getAllIdsFromChannel(channel, USER_EXPORT_MAX_ITEMS)
+ do {
+ videoIds = await VideoModel.getAllIdsByAccount({ account: this.user.Account, start, count: chunkSize })
+ start += videoIds.length
await Bluebird.map(videoIds, async id => {
try {
@@ -72,10 +73,10 @@ export class VideosExporter extends AbstractUserExporter<VideoExportJSON> {
logger.warn('Cannot export video %d.', id, { err })
}
}, { concurrency: 10 })
- }
+ } while (videoIds.length === chunkSize)
return {
- json: { videos: videosJSON },
+ json: { videos: sortBy(videosJSON, 'publishedAt') },
activityPubOutbox,
staticFiles
}
diff --git a/server/core/lib/video-channel.ts b/server/core/lib/video-channel.ts
index e4b01a68a1..33cc70c738 100644
--- a/server/core/lib/video-channel.ts
+++ b/server/core/lib/video-channel.ts
@@ -1,5 +1,5 @@
-import * as Sequelize from 'sequelize'
import { VideoChannelCreate } from '@peertube/peertube-models'
+import * as Sequelize from 'sequelize'
import { VideoChannelModel } from '../models/video/video-channel.js'
import { VideoModel } from '../models/video/video.js'
import { MAccountId, MChannelId } from '../types/models/index.js'
@@ -7,16 +7,16 @@ import { getLocalVideoChannelActivityPubUrl } from './activitypub/url.js'
import { federateVideoIfNeeded } from './activitypub/videos/index.js'
import { buildActorInstance } from './local-actor.js'
-async function createLocalVideoChannelWithoutKeys (videoChannelInfo: VideoChannelCreate, account: MAccountId, t: Sequelize.Transaction) {
- const url = getLocalVideoChannelActivityPubUrl(videoChannelInfo.name)
- const actorInstance = buildActorInstance('Group', url, videoChannelInfo.name)
+export async function createLocalVideoChannelWithoutKeys (body: VideoChannelCreate, account: MAccountId, t: Sequelize.Transaction) {
+ const url = getLocalVideoChannelActivityPubUrl(body.name)
+ const actorInstance = buildActorInstance('Group', url, body.name)
const actorInstanceCreated = await actorInstance.save({ transaction: t })
const videoChannelData = {
- name: videoChannelInfo.displayName,
- description: videoChannelInfo.description,
- support: videoChannelInfo.support,
+ name: body.displayName,
+ description: body.description,
+ support: body.support,
accountId: account.id,
actorId: actorInstanceCreated.id
}
@@ -32,8 +32,8 @@ async function createLocalVideoChannelWithoutKeys (videoChannelInfo: VideoChanne
return videoChannelCreated
}
-async function federateAllVideosOfChannel (videoChannel: MChannelId) {
- const videoIds = await VideoModel.getAllIdsFromChannel(videoChannel)
+export async function federateAllVideosOfChannel (videoChannel: MChannelId) {
+ const videoIds = await VideoModel.getAllIdsFromChannel({ videoChannel, count: 1000 })
for (const videoId of videoIds) {
const video = await VideoModel.loadFull(videoId)
@@ -41,10 +41,3 @@ async function federateAllVideosOfChannel (videoChannel: MChannelId) {
await federateVideoIfNeeded(video, false)
}
}
-
-// ---------------------------------------------------------------------------
-
-export {
- createLocalVideoChannelWithoutKeys,
- federateAllVideosOfChannel
-}
diff --git a/server/core/models/video/video.ts b/server/core/models/video/video.ts
index a66e8aecb2..90b558d6bf 100644
--- a/server/core/models/video/video.ts
+++ b/server/core/models/video/video.ts
@@ -82,6 +82,7 @@ import { CONFIG } from '../../initializers/config.js'
import { ACTIVITY_PUB, API_VERSION, CONSTRAINTS_FIELDS, WEBSERVER } from '../../initializers/constants.js'
import { sendDeleteVideo } from '../../lib/activitypub/send/index.js'
import {
+ MAccountId,
MChannel,
MChannelAccountDefault,
MChannelId,
@@ -1562,13 +1563,44 @@ export class VideoModel extends SequelizeModel<VideoModel> {
return VideoModel.update({ support: ofChannel.support }, options)
}
- static async getAllIdsFromChannel (videoChannel: MChannelId, limit?: number): Promise<number[]> {
+ static async getAllIdsFromChannel (options: {
+ videoChannel: MChannelId
+ count: number
+ }): Promise<number[]> {
+ const { videoChannel, count } = options
+
const videos = await VideoModel.findAll({
attributes: [ 'id' ],
where: {
channelId: videoChannel.id
},
- limit
+ limit: count
+ })
+
+ return videos.map(v => v.id)
+ }
+
+ static async getAllIdsByAccount (options: {
+ account: MAccountId
+ start: number
+ count: number
+ }): Promise<number[]> {
+ const { account, start, count } = options
+
+ const videos = await VideoModel.findAll({
+ attributes: [ 'id' ],
+ include: [
+ {
+ attributes: [ 'accountId' ],
+ model: VideoChannelModel.unscoped(),
+ required: true,
+ where: {
+ accountId: account.id
+ }
+ }
+ ],
+ offset: start,
+ limit: count
})
return videos.map(v => v.id)