Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Add ability to cleanup views on local videos too

Chocobozzz4fd4c91ec1fc
diff --git a/client/src/app/+videos-publish-manage/shared-manage/stats/video-stats.component.html b/client/src/app/+videos-publish-manage/shared-manage/stats/video-stats.component.html
index 4f80c2bd04..f4262f5c6f 100644
--- a/client/src/app/+videos-publish-manage/shared-manage/stats/video-stats.component.html
+++ b/client/src/app/+videos-publish-manage/shared-manage/stats/video-stats.component.html
@@ -25,7 +25,13 @@
<div class="stats-with-date">
<div class="overall-stats">
- <h2>{{ getViewersStatsTitle() }}</h2>
+ <div class="title-container mb-4">
+ <h3 class="mb-0">{{ getViewersStatsTitle() }}</h3>
+
+ @if (hasMaxViewsAge()) {
+ <div class="muted-2 small" i18n>Views made before {{ getMaxViewsAgeDate() | date }} are not taken into account</div>
+ }
+ </div>
<div class="date-filter-wrapper">
<label class="visually-hidden" for="date-filter">Filter viewers stats by date</label>
diff --git a/client/src/app/+videos-publish-manage/shared-manage/stats/video-stats.component.scss b/client/src/app/+videos-publish-manage/shared-manage/stats/video-stats.component.scss
index 0098a0e090..daac7906c4 100644
--- a/client/src/app/+videos-publish-manage/shared-manage/stats/video-stats.component.scss
+++ b/client/src/app/+videos-publish-manage/shared-manage/stats/video-stats.component.scss
@@ -1,6 +1,6 @@
-@use '_variables' as *;
-@use '_mixins' as *;
-@use 'nav' as *;
+@use "_variables" as *;
+@use "_mixins" as *;
+@use "nav" as *;
.stats-embed {
display: flex;
@@ -12,8 +12,12 @@
display: flex;
flex-wrap: wrap;
- h2 {
- font-size: 16px;
+ .title-container {
+ h3 {
+ font-size: 16px;
+ font-weight: $font-bold;
+ }
+
width: 100%;
}
}
diff --git a/client/src/app/+videos-publish-manage/shared-manage/stats/video-stats.component.ts b/client/src/app/+videos-publish-manage/shared-manage/stats/video-stats.component.ts
index a1f958958e..4e01b95c79 100644
--- a/client/src/app/+videos-publish-manage/shared-manage/stats/video-stats.component.ts
+++ b/client/src/app/+videos-publish-manage/shared-manage/stats/video-stats.component.ts
@@ -2,7 +2,7 @@ import { CommonModule } from '@angular/common'
import { Component, LOCALE_ID, OnInit, inject } from '@angular/core'
import { FormsModule } from '@angular/forms'
import { ActivatedRoute } from '@angular/router'
-import { Notifier, PeerTubeRouterService } from '@app/core'
+import { Notifier, PeerTubeRouterService, ServerService } from '@app/core'
import { GlobalIconComponent } from '@app/shared/shared-icons/global-icon.component'
import { NumberFormatterPipe } from '@app/shared/shared-main/common/number-formatter.pipe'
import { LiveVideoService } from '@app/shared/shared-video-live/live-video.service'
@@ -83,6 +83,7 @@ export class VideoStatsComponent implements OnInit {
private numberFormatter = inject(NumberFormatterPipe)
private liveService = inject(LiveVideoService)
private manageController = inject(VideoManageController)
+ private serverService = inject(ServerService)
// Cannot handle date filters
globalStatsCards: Card[] = []
@@ -751,4 +752,18 @@ export class VideoStatsComponent implements OnInit {
second: 'numeric'
})
}
+
+ // ---------------------------------------------------------------------------
+
+ hasMaxViewsAge () {
+ return this.getMaxViewsAge() !== -1
+ }
+
+ getMaxViewsAgeDate () {
+ return new Date(Date.now() - this.getMaxViewsAge())
+ }
+
+ private getMaxViewsAge () {
+ return this.serverService.getHTMLConfig().views.videos.local.maxAge
+ }
}
diff --git a/config/default.yaml b/config/default.yaml
index fc3cfb71ec..f3acfac161 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -362,13 +362,21 @@ history:
views:
videos:
# PeerTube creates a database entry every hour for each video to track views over a period of time
- # This is used in particular by the Trending page
- # PeerTube could remove old remote video views if you want to reduce your database size (video view counter will not be altered)
+ # This is used in particular by the Trending/Hot algorithms
+ # PeerTube can remove views from remote videos if you want to reduce your database size (video view counter will not be altered)
# -1 means no cleanup
# Other values could be '6 months' or '30 days' etc (PeerTube will periodically delete old entries from database)
remote:
max_age: '30 days'
+ # PeerTube can also remove views informations from local videos
+ # Local views are used by the Trending/Hot algorithms, as remote views, but they are also used to display view stats to video makers
+ # Video view counter will not be altered, but video makers won't be able to see views stats of their videos before "max_age"
+ # -1 means no cleanup
+ # Other values could be '6 months' or '30 days' etc (PeerTube will periodically delete old entries from database)
+ local:
+ max_age: -1
+
# PeerTube buffers local video views before updating and federating the video
local_buffer_update_interval: '30 minutes'
diff --git a/config/dev.yaml b/config/dev.yaml
index 85114f824d..2398572d0c 100644
--- a/config/dev.yaml
+++ b/config/dev.yaml
@@ -129,6 +129,9 @@ views:
remote:
max_age: -1
+ local:
+ max_age: '15 years'
+
watching_interval:
anonymous: '6 seconds'
users: '4 seconds'
diff --git a/config/production.yaml.example b/config/production.yaml.example
index 62bce09600..03a7abcf66 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -360,13 +360,21 @@ history:
views:
videos:
# PeerTube creates a database entry every hour for each video to track views over a period of time
- # This is used in particular by the Trending page
- # PeerTube could remove old remote video views if you want to reduce your database size (video view counter will not be altered)
+ # This is used in particular by the Trending/Hot algorithms
+ # PeerTube can remove views from remote videos if you want to reduce your database size (video view counter will not be altered)
# -1 means no cleanup
# Other values could be '6 months' or '30 days' etc (PeerTube will periodically delete old entries from database)
remote:
max_age: '30 days'
+ # PeerTube can also remove views informations from local videos
+ # Local views are used by the Trending/Hot algorithms, as remote views, but they are also used to display view stats to video makers
+ # Video view counter will not be altered, but video makers won't be able to see views stats of their videos before "max_age"
+ # -1 means no cleanup
+ # Other values could be '6 months' or '30 days' etc (PeerTube will periodically delete old entries from database)
+ local:
+ max_age: -1
+
# PeerTube buffers local video views before updating and federating the video
local_buffer_update_interval: '30 minutes'
diff --git a/packages/models/src/server/debug.model.ts b/packages/models/src/server/debug.model.ts
index ea43d4faaf..ffab325ef1 100644
--- a/packages/models/src/server/debug.model.ts
+++ b/packages/models/src/server/debug.model.ts
@@ -11,6 +11,7 @@ export type SendDebugCommand = {
| 'process-video-channel-sync-latest'
| 'process-update-videos-scheduler'
| 'remove-expired-user-exports'
+ | 'process-remove-old-views'
} | SendDebugTestEmails
export type SendDebugTestEmails = {
diff --git a/packages/models/src/server/server-config.model.ts b/packages/models/src/server/server-config.model.ts
index 0a3478c9c0..1332470edf 100644
--- a/packages/models/src/server/server-config.model.ts
+++ b/packages/models/src/server/server-config.model.ts
@@ -408,6 +408,16 @@ export interface ServerConfig {
views: {
videos: {
+ remote: {
+ // milliseconds
+ maxAge: number
+ }
+
+ local: {
+ // milliseconds
+ maxAge: number
+ }
+
watchingInterval: {
// milliseconds
anonymous: number
diff --git a/packages/tests/src/api/views/videos-views-cleaner.ts b/packages/tests/src/api/views/videos-views-cleaner.ts
index 1246b90b44..9d087c775f 100644
--- a/packages/tests/src/api/views/videos-views-cleaner.ts
+++ b/packages/tests/src/api/views/videos-views-cleaner.ts
@@ -1,6 +1,5 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
-import { wait } from '@peertube/peertube-core-utils'
import { buildUUID } from '@peertube/peertube-node-utils'
import {
cleanupTests,
@@ -12,6 +11,7 @@ import {
waitJobs
} from '@peertube/peertube-server-commands'
import { SQLCommand } from '@tests/shared/sql-command.js'
+import { processViewersStats } from '@tests/shared/views.js'
import { expect } from 'chai'
describe('Test video views cleaner', function () {
@@ -40,55 +40,118 @@ describe('Test video views cleaner', function () {
await servers[1].views.simulateView({ id: videoIdServer1, sessionId })
await servers[0].views.simulateView({ id: videoIdServer2, sessionId })
await servers[1].views.simulateView({ id: videoIdServer2, sessionId })
+ await processViewersStats(servers)
await waitJobs(servers)
sqlCommands = servers.map(s => new SQLCommand(s))
})
- it('Should not clean old video views', async function () {
- this.timeout(50000)
+ describe('Views on remote videos', function () {
+ it('Should not clean old video views', async function () {
+ this.timeout(50000)
- await killallServers([ servers[0] ])
+ await killallServers([ servers[0] ])
+ await servers[0].run({ views: { videos: { remote: { max_age: '10 days' } } } })
+ await servers[0].debug.sendCommand({ body: { command: 'process-remove-old-views' } })
- await servers[0].run({ views: { videos: { remote: { max_age: '10 days' } } } })
+ for (let i = 0; i < servers.length; i++) {
+ const total = await sqlCommands[i].countVideoViewsOf(videoIdServer1)
+ expect(total).to.equal(2, 'Server ' + servers[i].serverNumber + ' does not have the correct amount of views')
+ }
- await wait(6000)
+ for (let i = 0; i < servers.length; i++) {
+ const total = await sqlCommands[i].countVideoViewsOf(videoIdServer2)
+ expect(total).to.equal(2, 'Server ' + servers[i].serverNumber + ' does not have the correct amount of views')
+ }
- // Should still have views
+ const stats = await servers[0].videoStats.getOverallStats({ videoId: videoIdServer1 })
+ expect(stats.totalViewers).to.equal(2)
+ expect(stats.totalWatchTime).to.equal(10)
+ })
- for (let i = 0; i < servers.length; i++) {
- const total = await sqlCommands[i].countVideoViewsOf(videoIdServer1)
- expect(total).to.equal(2, 'Server ' + servers[i].serverNumber + ' does not have the correct amount of views')
- }
+ it('Should clean old video views', async function () {
+ this.timeout(50000)
- for (let i = 0; i < servers.length; i++) {
- const total = await sqlCommands[i].countVideoViewsOf(videoIdServer2)
- expect(total).to.equal(2, 'Server ' + servers[i].serverNumber + ' does not have the correct amount of views')
- }
+ await killallServers([ servers[0] ])
+ await servers[0].run({ views: { videos: { remote: { max_age: '5 seconds' } } } })
+ await servers[0].debug.sendCommand({ body: { command: 'process-remove-old-views' } })
+
+ for (let i = 0; i < servers.length; i++) {
+ const total = await sqlCommands[i].countVideoViewsOf(videoIdServer1)
+ expect(total).to.equal(2)
+ }
+
+ const totalServer1 = await sqlCommands[0].countVideoViewsOf(videoIdServer2)
+ expect(totalServer1).to.equal(0)
+
+ const totalServer2 = await sqlCommands[1].countVideoViewsOf(videoIdServer2)
+ expect(totalServer2).to.equal(2)
+
+ const stats = await servers[0].videoStats.getOverallStats({ videoId: videoIdServer1 })
+ expect(stats.totalViewers).to.equal(2)
+ expect(stats.totalWatchTime).to.equal(10)
+ })
})
- it('Should clean old video views', async function () {
- this.timeout(50000)
+ describe('Views on local videos', function () {
+ it('Should not clean old video views', async function () {
+ this.timeout(50000)
- await killallServers([ servers[0] ])
+ await killallServers([ servers[0] ])
+ await servers[0].run({ views: { videos: { local: { max_age: '5 hours' } } } })
+ await servers[0].debug.sendCommand({ body: { command: 'process-remove-old-views' } })
- await servers[0].run({ views: { videos: { remote: { max_age: '5 seconds' } } } })
+ for (let i = 0; i < servers.length; i++) {
+ const total = await sqlCommands[i].countVideoViewsOf(videoIdServer1)
+ expect(total).to.equal(2)
+ }
- await wait(6000)
+ const totalServer1 = await sqlCommands[0].countVideoViewsOf(videoIdServer2)
+ expect(totalServer1).to.equal(0)
- // Should still have views
+ const totalServer2 = await sqlCommands[1].countVideoViewsOf(videoIdServer2)
+ expect(totalServer2).to.equal(2)
- for (let i = 0; i < servers.length; i++) {
- const total = await sqlCommands[i].countVideoViewsOf(videoIdServer1)
- expect(total).to.equal(2)
- }
+ const stats = await servers[0].videoStats.getOverallStats({ videoId: videoIdServer1 })
+ expect(stats.totalViewers).to.equal(2)
+ expect(stats.totalWatchTime).to.equal(10)
+ })
- const totalServer1 = await sqlCommands[0].countVideoViewsOf(videoIdServer2)
- expect(totalServer1).to.equal(0)
+ it('Should clean old video views', async function () {
+ this.timeout(50000)
+
+ await killallServers([ servers[0] ])
+ await servers[0].run({ views: { videos: { local: { max_age: '5 seconds' } } } })
+ await servers[0].debug.sendCommand({ body: { command: 'process-remove-old-views' } })
+
+ {
+ const totalServer1 = await sqlCommands[0].countVideoViewsOf(videoIdServer1)
+ expect(totalServer1).to.equal(0)
+
+ const totalServer2 = await sqlCommands[1].countVideoViewsOf(videoIdServer1)
+ expect(totalServer2).to.equal(2)
+ }
+
+ {
+ const totalServer1 = await sqlCommands[0].countVideoViewsOf(videoIdServer2)
+ expect(totalServer1).to.equal(0)
+
+ const totalServer2 = await sqlCommands[1].countVideoViewsOf(videoIdServer2)
+ expect(totalServer2).to.equal(2)
+ }
+
+ const stats = await servers[0].videoStats.getOverallStats({ videoId: videoIdServer1 })
+ expect(stats.totalViewers).to.equal(0)
+ expect(stats.totalWatchTime).to.equal(0)
+ })
+ })
- const totalServer2 = await sqlCommands[1].countVideoViewsOf(videoIdServer2)
- expect(totalServer2).to.equal(2)
+ describe('Views counter', function () {
+ it('Should still have the appropriate views counter', async function () {
+ const { views } = await servers[0].videos.get({ id: videoIdServer1 })
+ expect(views).to.equal(2)
+ })
})
after(async function () {
diff --git a/packages/tests/src/server-helpers/core-utils.ts b/packages/tests/src/server-helpers/core-utils.ts
index 514caeb760..15129604ea 100644
--- a/packages/tests/src/server-helpers/core-utils.ts
+++ b/packages/tests/src/server-helpers/core-utils.ts
@@ -61,6 +61,7 @@ describe('Parse duration', function () {
expect(parseDurationToMs('1 minute')).to.equal(60 * 1000)
expect(parseDurationToMs('1 hour')).to.equal(3600 * 1000)
expect(parseDurationToMs('35 hours')).to.equal(3600 * 35 * 1000)
+ expect(parseDurationToMs('15 years')).to.equal(15 * 3600 * 24 * 365 * 1000)
})
it('Should be invalid when given invalid value', function () {
diff --git a/server/core/controllers/api/server/debug.ts b/server/core/controllers/api/server/debug.ts
index d49395238c..f843eb597e 100644
--- a/server/core/controllers/api/server/debug.ts
+++ b/server/core/controllers/api/server/debug.ts
@@ -21,6 +21,7 @@ import { VideoViewsBufferScheduler } from '@server/lib/schedulers/video-views-bu
import { VideoViewsManager } from '@server/lib/views/video-views-manager.js'
import express from 'express'
import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares/index.js'
+import { RemoveOldViewsScheduler } from '@server/lib/schedulers/remove-old-views-scheduler.js'
const debugRouter = express.Router()
@@ -63,6 +64,7 @@ async function runCommand (req: express.Request, res: express.Response) {
'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(),
'process-update-videos-scheduler': () => UpdateVideosScheduler.Instance.execute(),
'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute(),
+ 'process-remove-old-views': () => RemoveOldViewsScheduler.Instance.execute(),
'test-emails': () => testEmails(req, res)
}
diff --git a/server/core/helpers/core-utils.ts b/server/core/helpers/core-utils.ts
index c52d7ddd69..2288f03e68 100644
--- a/server/core/helpers/core-utils.ts
+++ b/server/core/helpers/core-utils.ts
@@ -50,7 +50,8 @@ const timeTable = {
hour: 3600000,
day: 3600000 * 24,
week: 3600000 * 24 * 7,
- month: 3600000 * 24 * 30
+ month: 3600000 * 24 * 30,
+ year: 3600000 * 24 * 365
}
export function parseDurationToMs (duration: number | string): number {
@@ -68,7 +69,10 @@ export function parseDurationToMs (duration: number | string): number {
unit = 'ms'
}
- return (len || 1) * (timeTable[unit] || 0)
+ const multiplier = timeTable[unit]
+ if (!multiplier) throw new Error('Cannot parse datetime unit ' + unit)
+
+ return (len || 1) * multiplier
}
}
@@ -271,19 +275,19 @@ const pipelinePromise = promisify(pipeline)
// ---------------------------------------------------------------------------
export {
- objectConverter,
- mapToJSON,
- sanitizeUrl,
- sanitizeHost,
+ execPromise,
+ execPromise2,
execShell,
+ generateED25519KeyPairPromise,
+ generateRSAKeyPairPromise,
+ mapToJSON,
+ objectConverter,
pageToStartAndCount,
+ parseSemVersion,
peertubeTruncate,
- scryptPromise,
- randomBytesPromise,
- generateRSAKeyPairPromise,
- generateED25519KeyPairPromise,
- execPromise2,
- execPromise,
pipelinePromise,
- parseSemVersion
+ randomBytesPromise,
+ sanitizeHost,
+ sanitizeUrl,
+ scryptPromise
}
diff --git a/server/core/initializers/config.ts b/server/core/initializers/config.ts
index 84d7cddc56..d691ac07b5 100644
--- a/server/core/initializers/config.ts
+++ b/server/core/initializers/config.ts
@@ -398,6 +398,9 @@ const CONFIG = {
REMOTE: {
MAX_AGE: parseDurationToMs(config.get('views.videos.remote.max_age'))
},
+ LOCAL: {
+ MAX_AGE: parseDurationToMs(config.get('views.videos.local.max_age'))
+ },
LOCAL_BUFFER_UPDATE_INTERVAL: parseDurationToMs(config.get('views.videos.local_buffer_update_interval')),
VIEW_EXPIRATION: parseDurationToMs(config.get('views.videos.view_expiration')),
COUNT_VIEW_AFTER: parseDurationToMs(config.get<number>('views.videos.count_view_after')),
diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts
index faf76160e4..32b2effcdb 100644
--- a/server/core/initializers/constants.ts
+++ b/server/core/initializers/constants.ts
@@ -1200,6 +1200,10 @@ export const STATS_TIMESERIE = {
// ---------------------------------------------------------------------------
+export const MAX_SQL_DELETE_ITEMS = 10000
+
+// ---------------------------------------------------------------------------
+
// Special constants for a test instance
if (process.env.PRODUCTION_CONSTANTS !== 'true') {
if (isTestOrDevInstance()) {
@@ -1215,7 +1219,6 @@ if (process.env.PRODUCTION_CONSTANTS !== 'true') {
SCHEDULER_INTERVALS_MS.ACTOR_FOLLOW_SCORES = 1000
SCHEDULER_INTERVALS_MS.REMOVE_OLD_JOBS = 10000
SCHEDULER_INTERVALS_MS.REMOVE_OLD_HISTORY = 5000
- SCHEDULER_INTERVALS_MS.REMOVE_OLD_VIEWS = 5000
SCHEDULER_INTERVALS_MS.UPDATE_VIDEOS = 5000
SCHEDULER_INTERVALS_MS.AUTO_FOLLOW_INDEX_INSTANCES = 5000
SCHEDULER_INTERVALS_MS.UPDATE_INBOX_STATS = 5000
diff --git a/server/core/lib/schedulers/remove-old-views-scheduler.ts b/server/core/lib/schedulers/remove-old-views-scheduler.ts
index 7bd28b17bc..039354db47 100644
--- a/server/core/lib/schedulers/remove-old-views-scheduler.ts
+++ b/server/core/lib/schedulers/remove-old-views-scheduler.ts
@@ -1,3 +1,4 @@
+import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer.js'
import { VideoViewModel } from '@server/models/view/video-view.js'
import { logger } from '../../helpers/logger.js'
import { CONFIG } from '../../initializers/config.js'
@@ -5,7 +6,6 @@ import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
import { AbstractScheduler } from './abstract-scheduler.js'
export class RemoveOldViewsScheduler extends AbstractScheduler {
-
private static instance: AbstractScheduler
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_OLD_VIEWS
@@ -14,15 +14,32 @@ export class RemoveOldViewsScheduler extends AbstractScheduler {
super()
}
- protected internalExecute () {
- if (CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE === -1) return
+ protected async internalExecute () {
+ await this.removeRemoteViews()
+ await this.removeLocalViews()
+ }
+
+ private removeRemoteViews () {
+ if (CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE <= 0) return
- logger.info('Removing old videos views.')
+ logger.info('Removing old views from remote videos.')
const now = new Date()
const beforeDate = new Date(now.getTime() - CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE).toISOString()
- return VideoViewModel.removeOldRemoteViewsHistory(beforeDate)
+ return VideoViewModel.removeOldRemoteViews(beforeDate)
+ }
+
+ private async removeLocalViews () {
+ if (CONFIG.VIEWS.VIDEOS.LOCAL.MAX_AGE <= 0) return
+
+ logger.info('Removing old views from local videos.')
+
+ const now = new Date()
+ const beforeDate = new Date(now.getTime() - CONFIG.VIEWS.VIDEOS.LOCAL.MAX_AGE).toISOString()
+
+ await VideoViewModel.removeOldLocalViews(beforeDate)
+ await LocalVideoViewerModel.removeOldViews(beforeDate)
}
static get Instance () {
diff --git a/server/core/lib/server-config-manager.ts b/server/core/lib/server-config-manager.ts
index 21fc3a5032..bf5aba156c 100644
--- a/server/core/lib/server-config-manager.ts
+++ b/server/core/lib/server-config-manager.ts
@@ -382,6 +382,14 @@ class ServerConfigManager {
views: {
videos: {
+ remote: {
+ maxAge: CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE
+ },
+
+ local: {
+ maxAge: CONFIG.VIEWS.VIDEOS.LOCAL.MAX_AGE
+ },
+
watchingInterval: {
anonymous: CONFIG.VIEWS.VIDEOS.WATCHING_INTERVAL.ANONYMOUS,
users: CONFIG.VIEWS.VIDEOS.WATCHING_INTERVAL.USERS
diff --git a/server/core/middlewares/validators/videos/video-stats.ts b/server/core/middlewares/validators/videos/video-stats.ts
index d9b3d9cccf..60ae29c707 100644
--- a/server/core/middlewares/validators/videos/video-stats.ts
+++ b/server/core/middlewares/validators/videos/video-stats.ts
@@ -75,7 +75,7 @@ export const videoTimeseriesStatsValidator = [
if (query.startDate && getIntervalByDays(query.startDate, query.endDate) > STATS_TIMESERIE.MAX_DAYS) {
return res.fail({
status: HttpStatusCode.BAD_REQUEST_400,
- message: 'Star date and end date interval is too big'
+ message: 'Start date and end date interval is too big'
})
}
diff --git a/server/core/models/shared/sql.ts b/server/core/models/shared/sql.ts
index 7520f5dce5..98b8641a3e 100644
--- a/server/core/models/shared/sql.ts
+++ b/server/core/models/shared/sql.ts
@@ -1,6 +1,7 @@
import { forceNumber } from '@peertube/peertube-core-utils'
import { FollowState } from '@peertube/peertube-models'
import { AttributesOnly } from '@peertube/peertube-typescript-utils'
+import { MAX_SQL_DELETE_ITEMS } from '@server/initializers/constants.js'
import { literal, Model, ModelStatic } from 'sequelize'
import { Literal } from 'sequelize/types/utils'
@@ -72,3 +73,11 @@ export function buildSQLAttributes<M extends Model> (options: {
return builtAttributes
}
+
+export async function safeBulkDestroy (destroyFn: () => Promise<number>) {
+ const destroyedRows = await destroyFn()
+
+ if (destroyedRows === MAX_SQL_DELETE_ITEMS) {
+ return safeBulkDestroy(destroyFn)
+ }
+}
diff --git a/server/core/models/view/local-video-viewer.ts b/server/core/models/view/local-video-viewer.ts
index fa39f7e344..160d2cd09a 100644
--- a/server/core/models/view/local-video-viewer.ts
+++ b/server/core/models/view/local-video-viewer.ts
@@ -6,12 +6,13 @@ import {
VideoStatsUserAgent,
WatchActionObject
} from '@peertube/peertube-models'
+import { MAX_SQL_DELETE_ITEMS } from '@server/initializers/constants.js'
import { getActivityStreamDuration } from '@server/lib/activitypub/activity.js'
import { buildGroupByAndBoundaries } from '@server/lib/timeserie.js'
import { MLocalVideoViewer, MLocalVideoViewerWithWatchSections, MVideo } from '@server/types/models/index.js'
-import { QueryOptionsWithType, QueryTypes } from 'sequelize'
+import { Op, QueryOptionsWithType, QueryTypes } from 'sequelize'
import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, HasMany, IsUUID, Table } from 'sequelize-typescript'
-import { SequelizeModel } from '../shared/index.js'
+import { safeBulkDestroy, SequelizeModel } from '../shared/index.js'
import { VideoModel } from '../video/video.js'
import { LocalVideoViewerWatchSectionModel } from './local-video-viewer-watch-section.js'
@@ -434,4 +435,19 @@ export class LocalVideoViewerModel extends SequelizeModel<LocalVideoViewerModel>
...location
}
}
+
+ // ---------------------------------------------------------------------------
+
+ static removeOldViews (beforeDate: string) {
+ return safeBulkDestroy(() => {
+ return LocalVideoViewerModel.destroy({
+ where: {
+ startDate: {
+ [Op.lt]: beforeDate
+ }
+ },
+ limit: MAX_SQL_DELETE_ITEMS
+ })
+ })
+ }
}
diff --git a/server/core/models/view/video-view.ts b/server/core/models/view/video-view.ts
index c4239b1d6b..cad6326c52 100644
--- a/server/core/models/view/video-view.ts
+++ b/server/core/models/view/video-view.ts
@@ -1,7 +1,8 @@
+import { MAX_SQL_DELETE_ITEMS } from '@server/initializers/constants.js'
import { literal, Op } from 'sequelize'
import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table } from 'sequelize-typescript'
+import { safeBulkDestroy, SequelizeModel } from '../shared/index.js'
import { VideoModel } from '../video/video.js'
-import { SequelizeModel } from '../shared/index.js'
/**
* Aggregate views of all videos federated with our instance
@@ -48,18 +49,35 @@ export class VideoViewModel extends SequelizeModel<VideoViewModel> {
})
declare Video: Awaited<VideoModel>
- static removeOldRemoteViewsHistory (beforeDate: string) {
- const query = {
- where: {
- startDate: {
- [Op.lt]: beforeDate
+ static removeOldRemoteViews (beforeDate: string) {
+ return safeBulkDestroy(() => {
+ return VideoViewModel.destroy({
+ where: {
+ startDate: {
+ [Op.lt]: beforeDate
+ },
+ videoId: {
+ [Op.in]: literal('(SELECT "id" FROM "video" WHERE "remote" IS TRUE)')
+ }
},
- videoId: {
- [Op.in]: literal('(SELECT "id" FROM "video" WHERE "remote" IS TRUE)')
- }
- }
- }
+ limit: MAX_SQL_DELETE_ITEMS
+ })
+ })
+ }
- return VideoViewModel.destroy(query)
+ static removeOldLocalViews (beforeDate: string) {
+ return safeBulkDestroy(() => {
+ return VideoViewModel.destroy({
+ where: {
+ startDate: {
+ [Op.lt]: beforeDate
+ },
+ videoId: {
+ [Op.in]: literal('(SELECT "id" FROM "video" WHERE "remote" IS FALSE)')
+ }
+ },
+ limit: MAX_SQL_DELETE_ITEMS
+ })
+ })
}
}