Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Add `contentUrl` for SEO

Chocobozzz41db7da56424
diff --git a/packages/tests/src/client/jsonld.ts b/packages/tests/src/client/jsonld.ts
index 6a6d7ff9bc..712b516ce2 100644
--- a/packages/tests/src/client/jsonld.ts
+++ b/packages/tests/src/client/jsonld.ts
@@ -1,7 +1,14 @@
/* oxlint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { maxBy, minBy } from '@peertube/peertube-core-utils'
-import { HttpStatusCode, VideoCreateResult, VideoPlaylistCreateResult, VideoPrivacy } from '@peertube/peertube-models'
+import {
+ HttpStatusCode,
+ VideoCreateResult,
+ VideoDetails,
+ VideoPlaylistCreateResult,
+ VideoPrivacy,
+ VideoResolution
+} from '@peertube/peertube-models'
import {
PeerTubeServer,
cleanupTests,
@@ -33,6 +40,14 @@ async function getJSONLD (server: PeerTubeServer, path: string) {
return JSON.parse(matches[1])
}
+// Web video file the closest to 720p, or the HLS master playlist for HLS only videos
+function getExpectedContentUrl (video: VideoDetails) {
+ const files = video.files.filter(f => f.resolution.id !== VideoResolution.H_NOVIDEO)
+ if (files.length === 0) return video.streamingPlaylists[0]?.playlistUrl
+
+ return minBy(files.map(f => ({ fileUrl: f.fileUrl, distance: Math.abs(f.resolution.id - 720) })), 'distance').fileUrl
+}
+
describe('Test JSONLD HTML tags', function () {
let servers: PeerTubeServer[]
@@ -121,6 +136,7 @@ describe('Test JSONLD HTML tags', function () {
'duration': 'PT' + video.duration + 'S',
'url': server.url + '/w/' + publicVideo.shortUUID,
'embedUrl': server.url + '/videos/embed/' + publicVideo.shortUUID,
+ 'contentUrl': getExpectedContentUrl(video),
'author': {
'@type': 'Organization',
@@ -206,6 +222,42 @@ describe('Test JSONLD HTML tags', function () {
])
}
})
+
+ it('Should prefer the 720p web video file as content URL', async function () {
+ this.timeout(120000)
+
+ await servers[0].config.enableMinimumTranscoding({ webVideo: true, hls: false })
+
+ const { uuid } = await servers[0].videos.quickUpload({ name: 'transcoded video' })
+ await waitJobs(servers)
+
+ for (const server of servers) {
+ const video = await server.videos.get({ id: uuid })
+ expect(video.files.map(f => f.resolution.id)).to.have.members([ 240, 720 ])
+
+ const jsonld = await getJSONLD(server, getWatchVideoBasePaths()[0] + uuid)
+ expect(jsonld.contentUrl).to.equal(video.files.find(f => f.resolution.id === 720).fileUrl)
+ }
+ })
+
+ it('Should use the HLS playlist as content URL of an HLS only video', async function () {
+ this.timeout(120000)
+
+ await servers[0].config.enableMinimumTranscoding({ webVideo: false, hls: true })
+
+ const { uuid } = await servers[0].videos.quickUpload({ name: 'hls only video' })
+ await waitJobs(servers)
+
+ for (const server of servers) {
+ const video = await server.videos.get({ id: uuid })
+ expect(video.files).to.have.lengthOf(0)
+
+ const jsonld = await getJSONLD(server, getWatchVideoBasePaths()[0] + uuid)
+ expect(jsonld.contentUrl).to.equal(video.streamingPlaylists[0].playlistUrl)
+ }
+
+ await servers[0].config.disableTranscoding()
+ })
})
describe('Playlist', function () {
diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts
index f49c558e30..b936b820b8 100644
--- a/server/core/initializers/constants.ts
+++ b/server/core/initializers/constants.ts
@@ -1251,12 +1251,14 @@ export const MEMOIZE_TTL = {
LIVE_ABLE_TO_UPLOAD: 1000 * 60, // 1 minute
LIVE_CHECK_SOCKET_HEALTH: 1000 * 60, // 1 minute
GET_STATS_FOR_OPEN_TELEMETRY_METRICS: 1000 * 60, // 1 minute
- EMBED_HTML: 1000 * 10 // 10 seconds
+ EMBED_HTML: 1000 * 10, // 10 seconds
+ VIDEO_SEO: 1000 * 10 // 10 seconds
}
export const MEMOIZE_LENGTH = {
INFO_HASH_EXISTS: 200,
- VIDEO_DURATION: 200
+ VIDEO_DURATION: 200,
+ VIDEO_SEO: 200
}
export const totalCPUs = Math.max(cpus().length, 1)
@@ -1392,6 +1394,7 @@ if (process.env.PRODUCTION_CONSTANTS !== 'true') {
MEMOIZE_TTL.OVERVIEWS_SAMPLE = 3000
MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD = 3000
MEMOIZE_TTL.EMBED_HTML = 1
+ MEMOIZE_TTL.VIDEO_SEO = 1
OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD = 2
PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME = 5000
diff --git a/server/core/lib/html/shared/tags-html.ts b/server/core/lib/html/shared/tags-html.ts
index 0013999a39..0a1cbd4ce4 100644
--- a/server/core/lib/html/shared/tags-html.ts
+++ b/server/core/lib/html/shared/tags-html.ts
@@ -20,6 +20,7 @@ type JsonldSchema = {
url: string
embedUrl?: string
+ contentUrl?: string
uploadDate?: string
thumbnailUrl?: string
@@ -104,6 +105,8 @@ export type TagsOptions = {
language: string
tags: string[]
+ contentUrl?: string
+
views: number
likes: number
dislikes: number
@@ -379,6 +382,9 @@ export class TagsHtml {
}
]
+ // Google prefers `contentUrl` over `embedUrl` to fetch the video content
+ if (video.contentUrl) schema['contentUrl'] = video.contentUrl
+
if (video.duration) schema['duration'] = getActivityStreamDuration(video.duration)
if (video.language) schema['inLanguage'] = video.language
diff --git a/server/core/lib/html/shared/video-html.ts b/server/core/lib/html/shared/video-html.ts
index 47f0515271..8f43205952 100644
--- a/server/core/lib/html/shared/video-html.ts
+++ b/server/core/lib/html/shared/video-html.ts
@@ -1,5 +1,5 @@
import { addQueryParams, escapeHTML } from '@peertube/peertube-core-utils'
-import { HttpStatusCode, VideoPrivacy } from '@peertube/peertube-models'
+import { HttpStatusCode, VideoPrivacy, VideoResolution } from '@peertube/peertube-models'
import { Memoize } from '@server/helpers/memoize.js'
import { getVideoRSSFeeds } from '@server/lib/rss.js'
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
@@ -133,6 +133,8 @@ export class VideoHtml {
video: {
publishedAt: video.publishedAt.toISOString(),
duration: video.duration,
+
+ contentUrl: this.getContentUrl(video),
views: video.views,
language: video.language,
dislikes: video.dislikes,
@@ -163,6 +165,29 @@ export class VideoHtml {
}, { video })
}
+ // Crawlers prefer a URL they can fetch and probe themselves over the embed page
+ private static getContentUrl (video: MVideoSeo) {
+ const webVideoFile = this.getSEOWebVideoFile(video)
+ if (webVideoFile) return webVideoFile.getFileUrl(video)
+
+ // HLS only video: fallback on the master playlist
+ return video.getHLSPlaylist()?.getMasterPlaylistUrl(video)
+ }
+
+ // Prefer 720p: crawlers get enough to probe the video without us serving them our biggest file
+ private static getSEOWebVideoFile (video: MVideoSeo) {
+ const files = (video.VideoFiles || []).filter(f => f.resolution !== VideoResolution.H_NOVIDEO)
+ if (files.length === 0) return undefined
+
+ return files.sort((a, b) => {
+ const distance = Math.abs(a.resolution - 720) - Math.abs(b.resolution - 720)
+ if (distance !== 0) return distance
+
+ // prefer smaller file
+ return a.resolution - b.resolution
+ })[0]
+ }
+
private static getOEmbedUrl (video: MVideo, currentQuery: Record<string, string>) {
const base = WEBSERVER.URL + video.getWatchStaticPath()
diff --git a/server/core/models/video/sql/video/video-model-get-query-builder.ts b/server/core/models/video/sql/video/video-model-get-query-builder.ts
index 8676aee3bc..f2f756d9a5 100644
--- a/server/core/models/video/sql/video/video-model-get-query-builder.ts
+++ b/server/core/models/video/sql/video/video-model-get-query-builder.ts
@@ -24,7 +24,7 @@ export type GetType =
| 'video'
| 'seo'
-const videoFilesInclude = new Set<GetType>([ 'api', 'ap', 'full', 'account-blacklist-files', 'all-files' ])
+const videoFilesInclude = new Set<GetType>([ 'api', 'ap', 'full', 'account-blacklist-files', 'all-files', 'seo' ])
// Infohashes are only needed to build magnet URIs (API) and P2P media loader tags (AP)
const infohashesInclude = new Set<GetType>([ 'api', 'ap' ])
const captionsInclude = new Set<GetType>([ 'ap', 'seo' ])
diff --git a/server/core/models/video/video.ts b/server/core/models/video/video.ts
index 14f5790758..f8d00f6d34 100644
--- a/server/core/models/video/video.ts
+++ b/server/core/models/video/video.ts
@@ -24,6 +24,7 @@ import {
} from '@peertube/peertube-models'
import { uuidToShort } from '@peertube/peertube-node-utils'
import { AttributesOnly } from '@peertube/peertube-typescript-utils'
+import { Memoize } from '@server/helpers/memoize.js'
import { getPrivaciesForFederation } from '@server/helpers/video.js'
import { MVideoToFederate, isPrivacyForFederation } from '@server/lib/activitypub/videos/federate.js'
import { InternalEventEmitter } from '@server/lib/internal-event-emitter.js'
@@ -83,7 +84,7 @@ import {
} from '../../helpers/custom-validators/videos.js'
import { createLogger } from '../../helpers/logger.js'
import { CONFIG } from '../../initializers/config.js'
-import { ACTIVITY_PUB, CONSTRAINTS_FIELDS, WEBSERVER } from '../../initializers/constants.js'
+import { ACTIVITY_PUB, CONSTRAINTS_FIELDS, MEMOIZE_LENGTH, MEMOIZE_TTL, WEBSERVER } from '../../initializers/constants.js'
import { sendDeleteVideo } from '../../lib/activitypub/send/index.js'
import type {
MAccountId,
@@ -1421,10 +1422,13 @@ export class VideoModel extends SequelizeModel<VideoModel> {
return queryBuilder.queryVideo({ id, transaction, type: 'blacklist' })
}
- static loadForSEO (id: number | string, transaction?: Transaction): Promise<MVideoSeo> {
+ // Watch/embed page HTML is rebuilt on every request: cache the video so a burst of hits on the same video
+ // doesn't replay the SEO queries (video + web video files + streaming playlist files)
+ @Memoize({ promise: true, max: MEMOIZE_LENGTH.VIDEO_SEO, maxAge: MEMOIZE_TTL.VIDEO_SEO })
+ static loadForSEO (id: number | string): Promise<MVideoSeo> {
const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize)
- return queryBuilder.queryVideo({ id, transaction, type: 'seo' })
+ return queryBuilder.queryVideo({ id, type: 'seo' })
}
static loadImmutableAttributes (id: number | string, t?: Transaction): Promise<MVideoImmutable> {
diff --git a/server/core/types/models/video/video.ts b/server/core/types/models/video/video.ts
index 5b3644fd1d..b18d05100c 100644
--- a/server/core/types/models/video/video.ts
+++ b/server/core/types/models/video/video.ts
@@ -115,6 +115,8 @@ export type MVideoSeo =
& Use<'VideoChannel', MChannelAccountLight>
& Use<'Tags', MTag[]>
& Use<'VideoCaptions', MVideoCaptionLanguageUrl[]>
+ & Use<'VideoFiles', MVideoFile[]>
+ & Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]>
// ############################################################################