ActivityPub-federated video streaming platform using P2P directly in your web browser
feat: add lastmod to sitemap entries for videos, channels and accounts (#7738)
valeron90ea9132115048
diff --git a/packages/tests/src/misc-endpoints.ts b/packages/tests/src/misc-endpoints.tsindex ddd09a0fa6..8f25305cfb 100644--- a/packages/tests/src/misc-endpoints.ts+++ b/packages/tests/src/misc-endpoints.ts@@ -1,5 +1,6 @@/* oxlint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */+import { wait } from '@peertube/peertube-core-utils'import { HttpStatusCode, VideoPrivacy } from '@peertube/peertube-models'import { uuidToShort } from '@peertube/peertube-node-utils'import {@@ -167,6 +168,14 @@ describe('Test misc endpoints', function () {describe('Test bots endpoints', function () {let uuid1: string+ let user1Token: string++ function extractLastmod (text: string, loc: string) {+ const escapedLoc = loc.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')+ const match = new RegExp(`<loc>${escapedLoc}</loc><lastmod>([^<]+)</lastmod>`).exec(text)++ return match?.[1]+ }it('Should get the empty sitemap', async function () {const res = await makeGetRequest({@@ -193,7 +202,9 @@ describe('Test misc endpoints', function () {it('Should add videos, channel and accounts and get sitemap', async function () {this.timeout(35000)- const { token: user1Token } = await server.users.generate('user1')+ const user1 = await server.users.generate('user1')+ user1Token = user1.token+const { token: user2Token } = await server.users.generate('user2')const { token: user3Token } = await server.users.generate('user3')@@ -247,13 +258,105 @@ describe('Test misc endpoints', function () {expect(res.text).to.match(/<video:uploader.*>channel 1<\/video:uploader>/)expect(res.text).to.match(/<video:live>NO<\/video:live>/)- expect(res.text).to.contain('<url><loc>' + server.url + '/c/channel1/videos</loc></url>')- expect(res.text).to.contain('<url><loc>' + server.url + '/c/channel2/videos</loc></url>')+ expect(res.text).to.contain('<loc>' + server.url + '/c/channel1/videos</loc>')+ expect(res.text).to.contain('<loc>' + server.url + '/c/channel2/videos</loc>')expect(res.text).to.not.contain('<url><loc>' + server.url + '/c/channel3/videos</loc></url>')- expect(res.text).to.contain('<url><loc>' + server.url + '/a/user1/video-channels</loc></url>')- expect(res.text).to.contain('<url><loc>' + server.url + '/a/user2/video-channels</loc></url>')+ expect(res.text).to.contain('<loc>' + server.url + '/a/user1/video-channels</loc>')+ expect(res.text).to.contain('<loc>' + server.url + '/a/user2/video-channels</loc>')expect(res.text).to.not.contain('<url><loc>' + server.url + '/a/user3/video-channels</loc></url>')++ const isoDateRegexp = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/++ expect(extractLastmod(res.text, server.url + '/w/' + uuidToShort(uuid1))).to.match(isoDateRegexp)+ expect(extractLastmod(res.text, server.url + '/c/channel1/videos')).to.match(isoDateRegexp)+ expect(extractLastmod(res.text, server.url + '/a/user1/video-channels')).to.match(isoDateRegexp)+ })++ it('Should only update the video lastmod when relevant content changed', async function () {+ this.timeout(20000)++ const videoLoc = server.url + '/w/' + uuidToShort(uuid1)++ const getLastmod = async (query: string) => {+ const res = await makeGetRequest({+ url: server.url,+ path: '/sitemap.xml?t=' + query, // avoid using cache+ expectedStatus: HttpStatusCode.OK_200+ })++ return extractLastmod(res.text, videoLoc)+ }++ const initialLastmod = await getLastmod('lastmod-1')++ // Updating tags only should not change the lastmod value+ await server.videos.update({ id: uuid1, attributes: { tags: [ 'sea', 'bass' ] } })+ const afterTagsUpdateLastmod = await getLastmod('lastmod-2')+ expect(afterTagsUpdateLastmod).to.equal(initialLastmod)++ // Updating the name should bump the lastmod value+ await wait(1500)+ await server.videos.update({ id: uuid1, attributes: { name: 'video 1 updated' } })+ const afterNameUpdateLastmod = await getLastmod('lastmod-3')+ expect(afterNameUpdateLastmod).to.not.equal(initialLastmod)+ expect(new Date(afterNameUpdateLastmod).getTime()).to.be.greaterThan(new Date(initialLastmod).getTime())+ })++ it('Should update the channel lastmod when it is updated', async function () {+ this.timeout(20000)++ const channelLoc = server.url + '/c/channel1/videos'++ const res1 = await makeGetRequest({+ url: server.url,+ path: '/sitemap.xml?t=channel-lastmod-1',+ expectedStatus: HttpStatusCode.OK_200+ })+ const initialLastmod = extractLastmod(res1.text, channelLoc)++ await wait(1500)+ await server.channels.update({+ channelName: 'channel1',+ token: user1Token,+ attributes: { displayName: 'channel 1 updated' }+ })++ const res2 = await makeGetRequest({+ url: server.url,+ path: '/sitemap.xml?t=channel-lastmod-2',+ expectedStatus: HttpStatusCode.OK_200+ })+ const updatedLastmod = extractLastmod(res2.text, channelLoc)++ expect(updatedLastmod).to.not.equal(initialLastmod)+ expect(new Date(updatedLastmod).getTime()).to.be.greaterThan(new Date(initialLastmod).getTime())+ })++ it('Should update the account lastmod when it is updated', async function () {+ this.timeout(20000)++ const accountLoc = server.url + '/a/user1/video-channels'++ const res1 = await makeGetRequest({+ url: server.url,+ path: '/sitemap.xml?t=account-lastmod-1',+ expectedStatus: HttpStatusCode.OK_200+ })+ const initialLastmod = extractLastmod(res1.text, accountLoc)++ await wait(1500)+ await server.users.updateMe({ token: user1Token, description: 'updated bio' })++ const res2 = await makeGetRequest({+ url: server.url,+ path: '/sitemap.xml?t=account-lastmod-2',+ expectedStatus: HttpStatusCode.OK_200+ })+ const updatedLastmod = extractLastmod(res2.text, accountLoc)++ expect(updatedLastmod).to.not.equal(initialLastmod)+ expect(new Date(updatedLastmod).getTime()).to.be.greaterThan(new Date(initialLastmod).getTime())})it('Should not fail with big title/description videos', async function () {@@ -284,7 +387,7 @@ describe('Test misc endpoints', function () {})const url = server.url + '/w/' + uuidToShort(video.uuid)- expect(res.text).to.contain(`<url><loc>${url}</loc><video:video>`)+ expect(res.text).to.contain(`<loc>${url}</loc>`)}await server.kill()@@ -298,7 +401,7 @@ describe('Test misc endpoints', function () {})const url = server.url + '/videos/watch/' + video.uuid- expect(res.text).to.contain(`<url><loc>${url}</loc><video:video>`)+ expect(res.text).to.contain(`<loc>${url}</loc>`)}})})diff --git a/server/core/controllers/sitemap.ts b/server/core/controllers/sitemap.tsindex 4b7e5ac02a..2161e1fb0a 100644--- a/server/core/controllers/sitemap.ts+++ b/server/core/controllers/sitemap.ts@@ -58,13 +58,13 @@ async function getSitemap (req: express.Request, res: express.Response) {async function getSitemapVideoChannelUrls () {const rows = await VideoChannelModel.listLocalsForSitemap('createdAt')- return rows.map(channel => ({ url: channel.getClientUrl() }))+ return rows.map(channel => ({ url: channel.getClientUrl(), lastmod: channel.updatedAt.toISOString() }))}async function getSitemapAccountUrls () {const rows = await AccountModel.listLocalsForSitemap('createdAt')- return rows.map(account => ({ url: account.getClientUrl() }))+ return rows.map(account => ({ url: account.getClientUrl(), lastmod: account.updatedAt.toISOString() }))}async function getSitemapLocalVideoUrls () {@@ -105,6 +105,7 @@ async function getSitemapLocalVideoUrls () {url: process.env.EXPERIMENTAL_SITEMAP_VIDEO_URL === 'true'? WEBSERVER.URL + '/videos/watch/' + v.uuid: WEBSERVER.URL + v.getWatchStaticPath(),+ lastmod: (v.sitemapContentUpdatedAt ?? v.publishedAt).toISOString(),video: [{// Sitemap title should be < 100 charactersdiff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.tsindex 1ac2438361..ae8d9a1e8a 100644--- a/server/core/initializers/constants.ts+++ b/server/core/initializers/constants.ts@@ -62,7 +62,7 @@ import { CONFIG, registerConfigChangedHandler } from './config.js'// ----------------------------------------------------------------------------export const LAST_MIGRATION_VERSION = 1105+export const LAST_MIGRATION_VERSION = 1110// ---------------------------------------------------------------------------diff --git a/server/core/initializers/migrations/1110-video-content-updated-at.ts b/server/core/initializers/migrations/1110-video-content-updated-at.tsnew file mode 100644index 0000000000..cc5b6be021--- /dev/null+++ b/server/core/initializers/migrations/1110-video-content-updated-at.ts@@ -0,0 +1,26 @@+import * as Sequelize from 'sequelize'++async function up (utils: {+ transaction: Sequelize.Transaction+ queryInterface: Sequelize.QueryInterface+ sequelize: Sequelize.Sequelize+}): Promise<void> {+ await utils.sequelize.query(+ `ALTER TABLE "video" ADD COLUMN IF NOT EXISTS "sitemapContentUpdatedAt" TIMESTAMP WITH TIME ZONE`,+ { transaction: utils.transaction }+ )++ await utils.sequelize.query(+ `UPDATE "video" SET "sitemapContentUpdatedAt" = "updatedAt" WHERE "sitemapContentUpdatedAt" IS NULL`,+ { transaction: utils.transaction }+ )+}++function down (options) {+ throw new Error('Not implemented.')+}++export {+ down,+ up+}diff --git a/server/core/lib/local-video-updater.ts b/server/core/lib/local-video-updater.tsindex c783bb7c77..343a407a3a 100644--- a/server/core/lib/local-video-updater.ts+++ b/server/core/lib/local-video-updater.ts@@ -92,6 +92,11 @@ export class LocalVideoUpdater {if (options[key] !== undefined) video.set(key, options[key])}+ // Update sitemapContentUpdatedAt if relevant content fields changed+ if (this.hasSitemapContentChanged({ ...options, oldName, oldDescription })) {+ video.sitemapContentUpdatedAt = new Date()+ }+if (video.nsfw !== true) {video.nsfwFlags = NSFWFlag.NONEvideo.nsfwSummary = null@@ -393,4 +398,21 @@ export class LocalVideoUpdater {}]}++ private hasSitemapContentChanged (options: {+ name?: string+ description?: string+ thumbnails?: MThumbnail[]++ oldName: string+ oldDescription: string+ }) {+ const { name, description, thumbnails, oldName, oldDescription } = options++ const nameChanged = name !== undefined && name !== oldName+ const descriptionChanged = description !== undefined && description !== oldDescription+ const thumbnailChanged = thumbnails !== undefined && thumbnails.length !== 0++ return nameChanged || descriptionChanged || thumbnailChanged+ }}diff --git a/server/core/models/account/account.ts b/server/core/models/account/account.tsindex b133a40989..c3451802e5 100644--- a/server/core/models/account/account.ts+++ b/server/core/models/account/account.ts@@ -427,7 +427,7 @@ export class AccountModel extends SequelizeModel<AccountModel> {static listLocalsForSitemap (sort: string): Promise<MAccountHost[]> {return AccountModel.unscoped().findAll({- attributes: [],+ attributes: [ 'updatedAt' ],offset: 0,order: getSort(sort),include: [diff --git a/server/core/models/video/sql/video/shared/video-table-attributes.ts b/server/core/models/video/sql/video/shared/video-table-attributes.tsindex bd201f9df9..4e5bdab44e 100644--- a/server/core/models/video/sql/video/shared/video-table-attributes.ts+++ b/server/core/models/video/sql/video/shared/video-table-attributes.ts@@ -329,6 +329,7 @@ export class VideoTableAttributes {'originallyPublishedAt','inputFileUpdatedAt','firstPublishedAt',+ 'sitemapContentUpdatedAt','channelId','createdAt','updatedAt',diff --git a/server/core/models/video/video-channel.ts b/server/core/models/video/video-channel.tsindex 675ef08fb0..0f4a0c3c3a 100644--- a/server/core/models/video/video-channel.ts+++ b/server/core/models/video/video-channel.ts@@ -363,7 +363,7 @@ export class VideoChannelModel extends SequelizeModel<VideoChannelModel> {static listLocalsForSitemap (sort: string): Promise<MChannelHost[]> {const query = {- attributes: [],+ attributes: [ 'updatedAt' ],offset: 0,order: getSort(sort),include: [diff --git a/server/core/models/video/video.ts b/server/core/models/video/video.tsindex 3e50db4ff8..9cc525476c 100644--- a/server/core/models/video/video.ts+++ b/server/core/models/video/video.ts@@ -25,7 +25,7 @@ import {import { uuidToShort } from '@peertube/peertube-node-utils'import { AttributesOnly } from '@peertube/peertube-typescript-utils'import { getPrivaciesForFederation } from '@server/helpers/video.js'-import { isPrivacyForFederation, MVideoToFederate } from '@server/lib/activitypub/videos/federate.js'+import { MVideoToFederate, isPrivacyForFederation } from '@server/lib/activitypub/videos/federate.js'import { InternalEventEmitter } from '@server/lib/internal-event-emitter.js'import { LiveManager } from '@server/lib/live/live-manager.js'import {@@ -620,6 +620,11 @@ export class VideoModel extends SequelizeModel<VideoModel> {@Columndeclare firstPublishedAt: Date+ @AllowNull(true)+ @Default(null)+ @Column+ declare sitemapContentUpdatedAt: Date+@ForeignKey(() => VideoChannelModel)@Columndeclare channelId: number