Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

feat: add lastmod to sitemap entries for videos, channels and accounts (#7738)

valeron90ea9132115048
diff --git a/packages/tests/src/misc-endpoints.ts b/packages/tests/src/misc-endpoints.ts
index ddd09a0fa6..8f25305cfb 100644
--- a/packages/tests/src/misc-endpoints.ts
+++ b/packages/tests/src/misc-endpoints.ts
@@ -1,5 +1,6 @@
/* oxlint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+import { wait } from '@peertube/peertube-core-utils'
import { HttpStatusCode, VideoPrivacy } from '@peertube/peertube-models'
import { uuidToShort } from '@peertube/peertube-node-utils'
import {
@@ -167,6 +168,14 @@ describe('Test misc endpoints', function () {
describe('Test bots endpoints', function () {
let uuid1: string
+ let user1Token: string
+
+ function extractLastmod (text: string, loc: string) {
+ const escapedLoc = loc.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+ const match = new RegExp(`<loc>${escapedLoc}</loc><lastmod>([^<]+)</lastmod>`).exec(text)
+
+ return match?.[1]
+ }
it('Should get the empty sitemap', async function () {
const res = await makeGetRequest({
@@ -193,7 +202,9 @@ describe('Test misc endpoints', function () {
it('Should add videos, channel and accounts and get sitemap', async function () {
this.timeout(35000)
- const { token: user1Token } = await server.users.generate('user1')
+ const user1 = await server.users.generate('user1')
+ user1Token = user1.token
+
const { token: user2Token } = await server.users.generate('user2')
const { token: user3Token } = await server.users.generate('user3')
@@ -247,13 +258,105 @@ describe('Test misc endpoints', function () {
expect(res.text).to.match(/<video:uploader.*>channel 1<\/video:uploader>/)
expect(res.text).to.match(/<video:live>NO<\/video:live>/)
- expect(res.text).to.contain('<url><loc>' + server.url + '/c/channel1/videos</loc></url>')
- expect(res.text).to.contain('<url><loc>' + server.url + '/c/channel2/videos</loc></url>')
+ expect(res.text).to.contain('<loc>' + server.url + '/c/channel1/videos</loc>')
+ expect(res.text).to.contain('<loc>' + server.url + '/c/channel2/videos</loc>')
expect(res.text).to.not.contain('<url><loc>' + server.url + '/c/channel3/videos</loc></url>')
- expect(res.text).to.contain('<url><loc>' + server.url + '/a/user1/video-channels</loc></url>')
- expect(res.text).to.contain('<url><loc>' + server.url + '/a/user2/video-channels</loc></url>')
+ expect(res.text).to.contain('<loc>' + server.url + '/a/user1/video-channels</loc>')
+ expect(res.text).to.contain('<loc>' + server.url + '/a/user2/video-channels</loc>')
expect(res.text).to.not.contain('<url><loc>' + server.url + '/a/user3/video-channels</loc></url>')
+
+ const isoDateRegexp = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/
+
+ expect(extractLastmod(res.text, server.url + '/w/' + uuidToShort(uuid1))).to.match(isoDateRegexp)
+ expect(extractLastmod(res.text, server.url + '/c/channel1/videos')).to.match(isoDateRegexp)
+ expect(extractLastmod(res.text, server.url + '/a/user1/video-channels')).to.match(isoDateRegexp)
+ })
+
+ it('Should only update the video lastmod when relevant content changed', async function () {
+ this.timeout(20000)
+
+ const videoLoc = server.url + '/w/' + uuidToShort(uuid1)
+
+ const getLastmod = async (query: string) => {
+ const res = await makeGetRequest({
+ url: server.url,
+ path: '/sitemap.xml?t=' + query, // avoid using cache
+ expectedStatus: HttpStatusCode.OK_200
+ })
+
+ return extractLastmod(res.text, videoLoc)
+ }
+
+ const initialLastmod = await getLastmod('lastmod-1')
+
+ // Updating tags only should not change the lastmod value
+ await server.videos.update({ id: uuid1, attributes: { tags: [ 'sea', 'bass' ] } })
+ const afterTagsUpdateLastmod = await getLastmod('lastmod-2')
+ expect(afterTagsUpdateLastmod).to.equal(initialLastmod)
+
+ // Updating the name should bump the lastmod value
+ await wait(1500)
+ await server.videos.update({ id: uuid1, attributes: { name: 'video 1 updated' } })
+ const afterNameUpdateLastmod = await getLastmod('lastmod-3')
+ expect(afterNameUpdateLastmod).to.not.equal(initialLastmod)
+ expect(new Date(afterNameUpdateLastmod).getTime()).to.be.greaterThan(new Date(initialLastmod).getTime())
+ })
+
+ it('Should update the channel lastmod when it is updated', async function () {
+ this.timeout(20000)
+
+ const channelLoc = server.url + '/c/channel1/videos'
+
+ const res1 = await makeGetRequest({
+ url: server.url,
+ path: '/sitemap.xml?t=channel-lastmod-1',
+ expectedStatus: HttpStatusCode.OK_200
+ })
+ const initialLastmod = extractLastmod(res1.text, channelLoc)
+
+ await wait(1500)
+ await server.channels.update({
+ channelName: 'channel1',
+ token: user1Token,
+ attributes: { displayName: 'channel 1 updated' }
+ })
+
+ const res2 = await makeGetRequest({
+ url: server.url,
+ path: '/sitemap.xml?t=channel-lastmod-2',
+ expectedStatus: HttpStatusCode.OK_200
+ })
+ const updatedLastmod = extractLastmod(res2.text, channelLoc)
+
+ expect(updatedLastmod).to.not.equal(initialLastmod)
+ expect(new Date(updatedLastmod).getTime()).to.be.greaterThan(new Date(initialLastmod).getTime())
+ })
+
+ it('Should update the account lastmod when it is updated', async function () {
+ this.timeout(20000)
+
+ const accountLoc = server.url + '/a/user1/video-channels'
+
+ const res1 = await makeGetRequest({
+ url: server.url,
+ path: '/sitemap.xml?t=account-lastmod-1',
+ expectedStatus: HttpStatusCode.OK_200
+ })
+ const initialLastmod = extractLastmod(res1.text, accountLoc)
+
+ await wait(1500)
+ await server.users.updateMe({ token: user1Token, description: 'updated bio' })
+
+ const res2 = await makeGetRequest({
+ url: server.url,
+ path: '/sitemap.xml?t=account-lastmod-2',
+ expectedStatus: HttpStatusCode.OK_200
+ })
+ const updatedLastmod = extractLastmod(res2.text, accountLoc)
+
+ expect(updatedLastmod).to.not.equal(initialLastmod)
+ expect(new Date(updatedLastmod).getTime()).to.be.greaterThan(new Date(initialLastmod).getTime())
})
it('Should not fail with big title/description videos', async function () {
@@ -284,7 +387,7 @@ describe('Test misc endpoints', function () {
})
const url = server.url + '/w/' + uuidToShort(video.uuid)
- expect(res.text).to.contain(`<url><loc>${url}</loc><video:video>`)
+ expect(res.text).to.contain(`<loc>${url}</loc>`)
}
await server.kill()
@@ -298,7 +401,7 @@ describe('Test misc endpoints', function () {
})
const url = server.url + '/videos/watch/' + video.uuid
- expect(res.text).to.contain(`<url><loc>${url}</loc><video:video>`)
+ expect(res.text).to.contain(`<loc>${url}</loc>`)
}
})
})
diff --git a/server/core/controllers/sitemap.ts b/server/core/controllers/sitemap.ts
index 4b7e5ac02a..2161e1fb0a 100644
--- a/server/core/controllers/sitemap.ts
+++ b/server/core/controllers/sitemap.ts
@@ -58,13 +58,13 @@ async function getSitemap (req: express.Request, res: express.Response) {
async function getSitemapVideoChannelUrls () {
const rows = await VideoChannelModel.listLocalsForSitemap('createdAt')
- return rows.map(channel => ({ url: channel.getClientUrl() }))
+ return rows.map(channel => ({ url: channel.getClientUrl(), lastmod: channel.updatedAt.toISOString() }))
}
async function getSitemapAccountUrls () {
const rows = await AccountModel.listLocalsForSitemap('createdAt')
- return rows.map(account => ({ url: account.getClientUrl() }))
+ return rows.map(account => ({ url: account.getClientUrl(), lastmod: account.updatedAt.toISOString() }))
}
async function getSitemapLocalVideoUrls () {
@@ -105,6 +105,7 @@ async function getSitemapLocalVideoUrls () {
url: process.env.EXPERIMENTAL_SITEMAP_VIDEO_URL === 'true'
? WEBSERVER.URL + '/videos/watch/' + v.uuid
: WEBSERVER.URL + v.getWatchStaticPath(),
+ lastmod: (v.sitemapContentUpdatedAt ?? v.publishedAt).toISOString(),
video: [
{
// Sitemap title should be < 100 characters
diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts
index 1ac2438361..ae8d9a1e8a 100644
--- a/server/core/initializers/constants.ts
+++ b/server/core/initializers/constants.ts
@@ -62,7 +62,7 @@ import { CONFIG, registerConfigChangedHandler } from './config.js'
// ---------------------------------------------------------------------------
-export const LAST_MIGRATION_VERSION = 1105
+export const LAST_MIGRATION_VERSION = 1110
// ---------------------------------------------------------------------------
diff --git a/server/core/initializers/migrations/1110-video-content-updated-at.ts b/server/core/initializers/migrations/1110-video-content-updated-at.ts
new file mode 100644
index 0000000000..cc5b6be021
--- /dev/null
+++ b/server/core/initializers/migrations/1110-video-content-updated-at.ts
@@ -0,0 +1,26 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+ transaction: Sequelize.Transaction
+ queryInterface: Sequelize.QueryInterface
+ sequelize: Sequelize.Sequelize
+}): Promise<void> {
+ await utils.sequelize.query(
+ `ALTER TABLE "video" ADD COLUMN IF NOT EXISTS "sitemapContentUpdatedAt" TIMESTAMP WITH TIME ZONE`,
+ { transaction: utils.transaction }
+ )
+
+ await utils.sequelize.query(
+ `UPDATE "video" SET "sitemapContentUpdatedAt" = "updatedAt" WHERE "sitemapContentUpdatedAt" IS NULL`,
+ { transaction: utils.transaction }
+ )
+}
+
+function down (options) {
+ throw new Error('Not implemented.')
+}
+
+export {
+ down,
+ up
+}
diff --git a/server/core/lib/local-video-updater.ts b/server/core/lib/local-video-updater.ts
index c783bb7c77..343a407a3a 100644
--- a/server/core/lib/local-video-updater.ts
+++ b/server/core/lib/local-video-updater.ts
@@ -92,6 +92,11 @@ export class LocalVideoUpdater {
if (options[key] !== undefined) video.set(key, options[key])
}
+ // Update sitemapContentUpdatedAt if relevant content fields changed
+ if (this.hasSitemapContentChanged({ ...options, oldName, oldDescription })) {
+ video.sitemapContentUpdatedAt = new Date()
+ }
+
if (video.nsfw !== true) {
video.nsfwFlags = NSFWFlag.NONE
video.nsfwSummary = null
@@ -393,4 +398,21 @@ export class LocalVideoUpdater {
}
]
}
+
+ private hasSitemapContentChanged (options: {
+ name?: string
+ description?: string
+ thumbnails?: MThumbnail[]
+
+ oldName: string
+ oldDescription: string
+ }) {
+ const { name, description, thumbnails, oldName, oldDescription } = options
+
+ const nameChanged = name !== undefined && name !== oldName
+ const descriptionChanged = description !== undefined && description !== oldDescription
+ const thumbnailChanged = thumbnails !== undefined && thumbnails.length !== 0
+
+ return nameChanged || descriptionChanged || thumbnailChanged
+ }
}
diff --git a/server/core/models/account/account.ts b/server/core/models/account/account.ts
index b133a40989..c3451802e5 100644
--- a/server/core/models/account/account.ts
+++ b/server/core/models/account/account.ts
@@ -427,7 +427,7 @@ export class AccountModel extends SequelizeModel<AccountModel> {
static listLocalsForSitemap (sort: string): Promise<MAccountHost[]> {
return AccountModel.unscoped().findAll({
- attributes: [],
+ attributes: [ 'updatedAt' ],
offset: 0,
order: getSort(sort),
include: [
diff --git a/server/core/models/video/sql/video/shared/video-table-attributes.ts b/server/core/models/video/sql/video/shared/video-table-attributes.ts
index bd201f9df9..4e5bdab44e 100644
--- a/server/core/models/video/sql/video/shared/video-table-attributes.ts
+++ b/server/core/models/video/sql/video/shared/video-table-attributes.ts
@@ -329,6 +329,7 @@ export class VideoTableAttributes {
'originallyPublishedAt',
'inputFileUpdatedAt',
'firstPublishedAt',
+ 'sitemapContentUpdatedAt',
'channelId',
'createdAt',
'updatedAt',
diff --git a/server/core/models/video/video-channel.ts b/server/core/models/video/video-channel.ts
index 675ef08fb0..0f4a0c3c3a 100644
--- a/server/core/models/video/video-channel.ts
+++ b/server/core/models/video/video-channel.ts
@@ -363,7 +363,7 @@ export class VideoChannelModel extends SequelizeModel<VideoChannelModel> {
static listLocalsForSitemap (sort: string): Promise<MChannelHost[]> {
const query = {
- attributes: [],
+ attributes: [ 'updatedAt' ],
offset: 0,
order: getSort(sort),
include: [
diff --git a/server/core/models/video/video.ts b/server/core/models/video/video.ts
index 3e50db4ff8..9cc525476c 100644
--- a/server/core/models/video/video.ts
+++ b/server/core/models/video/video.ts
@@ -25,7 +25,7 @@ import {
import { uuidToShort } from '@peertube/peertube-node-utils'
import { AttributesOnly } from '@peertube/peertube-typescript-utils'
import { getPrivaciesForFederation } from '@server/helpers/video.js'
-import { isPrivacyForFederation, MVideoToFederate } from '@server/lib/activitypub/videos/federate.js'
+import { MVideoToFederate, isPrivacyForFederation } from '@server/lib/activitypub/videos/federate.js'
import { InternalEventEmitter } from '@server/lib/internal-event-emitter.js'
import { LiveManager } from '@server/lib/live/live-manager.js'
import {
@@ -620,6 +620,11 @@ export class VideoModel extends SequelizeModel<VideoModel> {
@Column
declare firstPublishedAt: Date
+ @AllowNull(true)
+ @Default(null)
+ @Column
+ declare sitemapContentUpdatedAt: Date
+
@ForeignKey(() => VideoChannelModel)
@Column
declare channelId: number