ActivityPub-federated video streaming platform using P2P directly in your web browser
Add watched words lists subscription
Chocobozzzb06c5bba5d13
diff --git a/AGENTS.md b/AGENTS.mdindex 9124e9202f..a7d77a41f3 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -11,8 +11,7 @@ handled via FFmpeg, with optional distributed runners.## Repository Structure- **apps/** — Standalone CLI applications (`peertube-cli`, `peertube-runner`)-- **client/** — Angular frontend SPA (separate build system; ignore for- backend work)+- **client/** — Angular frontend SPA (separate build system)- **config/** — YAML configuration files for dev, test, and production- **packages/** — Shared workspace packages (monorepo):- `core-utils/` — Shared pure-JS utilitiesdiff --git a/client/src/app/+admin/admin-moderation.component.ts b/client/src/app/+admin/admin-moderation.component.tsindex 666a7c9309..e03c5982e6 100644--- a/client/src/app/+admin/admin-moderation.component.ts+++ b/client/src/app/+admin/admin-moderation.component.ts@@ -82,10 +82,24 @@ export class AdminModerationComponent implements OnInit {}if (this.hasRight(UserRight.MANAGE_INSTANCE_WATCHED_WORDS)) {- this.menuEntries.push({+ const item: HorizontalMenuEntry = {label: $localize`Watched words`,- routerLink: '/admin/moderation/watched-words/list'- })+ routerLink: '',+ children: [+ {+ label: $localize`Lists`,+ routerLink: '/admin/moderation/watched-words/list'+ },+ {+ label: $localize`Subscriptions`,+ routerLink: '/admin/moderation/watched-words/subscriptions'+ }+ ]+ }++ item.routerLink = item.children[0].routerLink++ this.menuEntries.push(item)}}diff --git a/client/src/app/+admin/moderation/moderation.routes.ts b/client/src/app/+admin/moderation/moderation.routes.tsindex a1e652a6fa..39d1cbd34f 100644--- a/client/src/app/+admin/moderation/moderation.routes.ts+++ b/client/src/app/+admin/moderation/moderation.routes.ts@@ -10,6 +10,7 @@ import { UserRightGuard } from '@app/core'import { UserRight } from '@peertube/peertube-models'import { RegistrationListComponent } from './registration-list'import { WatchedWordsListAdminComponent } from './watched-words-list/watched-words-list-admin.component'+import { WatchedWordsSubscriptionsComponent } from './watched-words-subscriptions/watched-words-subscriptions.component'export const moderationRoutes: Routes = [{@@ -143,6 +144,18 @@ export const moderationRoutes: Routes = [title: $localize`Watched words`}}+ },++ {+ path: 'watched-words/subscriptions',+ component: WatchedWordsSubscriptionsComponent,+ canActivate: [ UserRightGuard ],+ data: {+ userRight: UserRight.MANAGE_INSTANCE_WATCHED_WORDS,+ meta: {+ title: $localize`Watched words subscriptions`+ }+ }}]}diff --git a/client/src/app/+admin/moderation/watched-words-list/watched-words-list-admin.component.html b/client/src/app/+admin/moderation/watched-words-list/watched-words-list-admin.component.htmlindex 370a11fc3b..6196ed6f85 100644--- a/client/src/app/+admin/moderation/watched-words-list/watched-words-list-admin.component.html+++ b/client/src/app/+admin/moderation/watched-words-list/watched-words-list-admin.component.html@@ -1,5 +1,7 @@-<em class="d-block" i18n>Video name/description and comments that contain any of the watched words are automatically tagged with the name of the list.</em>-<em class="d-block mb-3" i18n>These automatic tags can be used to filter comments and videos.</em>+<div class="callout callout-primary mb-4">+ <div i18n>Video name/description and comments that contain any of the watched words are automatically tagged with the name of the list.</div>+ <div i18n>These automatic tags can be used to filter comments and videos.</div>+</div><my-watched-words-list-admin-owner mode="admin"></my-watched-words-list-admin-owner>diff --git a/client/src/app/+admin/moderation/watched-words-subscriptions/watched-words-subscriptions.component.html b/client/src/app/+admin/moderation/watched-words-subscriptions/watched-words-subscriptions.component.htmlnew file mode 100644index 0000000000..0e0552249a--- /dev/null+++ b/client/src/app/+admin/moderation/watched-words-subscriptions/watched-words-subscriptions.component.html@@ -0,0 +1,5 @@+<div class="callout callout-primary mb-4" i18n>+ Subscribe to a remote watched words lists to automatically keep them synchronized for moderation.+</div>++<my-watched-words-subscriptions-admin-owner mode="admin"></my-watched-words-subscriptions-admin-owner>diff --git a/client/src/app/+admin/moderation/watched-words-subscriptions/watched-words-subscriptions.component.ts b/client/src/app/+admin/moderation/watched-words-subscriptions/watched-words-subscriptions.component.tsnew file mode 100644index 0000000000..74ab2daac6--- /dev/null+++ b/client/src/app/+admin/moderation/watched-words-subscriptions/watched-words-subscriptions.component.ts@@ -0,0 +1,13 @@+import { Component } from '@angular/core'+import { WatchedWordsSubscriptionService } from '@app/shared/shared-watched-words/watched-words-subscription.service'+import {+ WatchedWordsSubscriptionsAdminOwnerComponent+} from '@app/shared/shared-watched-words/watched-words-subscriptions-admin-owner.component'++@Component({+ templateUrl: './watched-words-subscriptions.component.html',+ imports: [ WatchedWordsSubscriptionsAdminOwnerComponent ],+ providers: [ WatchedWordsSubscriptionService ]+})+export class WatchedWordsSubscriptionsComponent {+}diff --git a/client/src/app/+admin/system/jobs/jobs.component.ts b/client/src/app/+admin/system/jobs/jobs.component.tsindex ec3084c15f..bcfa7934ab 100644--- a/client/src/app/+admin/system/jobs/jobs.component.ts+++ b/client/src/app/+admin/system/jobs/jobs.component.ts@@ -6,6 +6,7 @@ import { RestPagination, ServerService } from '@app/core'import { AdvancedFilterDef } from '@app/shared/shared-forms/advanced-input-filter.component'import { PeerTubeBadgeService } from '@app/shared/shared-main/common/peertube-badge.service'import { Job, JobState, JobType } from '@peertube/peertube-models'+import { arrayOfAllFactory } from '@peertube/peertube-typescript-utils'import { SortMeta } from 'primeng/api'import { NumberFormatterPipe } from '../../../shared/shared-main/common/number-formatter.pipe'import { TableColumnInfo, TableComponent } from '../../../shared/shared-tables/table.component'@@ -35,7 +36,7 @@ export class JobsComponent {private jobStates: JobState[] = [ 'active', 'completed', 'failed', 'waiting', 'delayed' ]- private jobTypes: JobType[] = [+ private jobTypes = arrayOfAllFactory<JobType>()(['activitypub-cleaner','activitypub-follow','activitypub-http-broadcast-parallel',@@ -45,10 +46,12 @@ export class JobsComponent {'activitypub-refresher','actor-keys','after-video-channel-import',+ 'build-automatic-tags','create-user-export','email','federate-video','generate-video-storyboard',+ 'import-user-archive','manage-video-torrent','move-to-file-system','move-to-object-storage',@@ -63,7 +66,7 @@ export class JobsComponent {'video-transcoding','video-transcription','videos-stats'- ]+ ])readonly inputFilters: AdvancedFilterDef<DataLoaderParameter>[] = [{diff --git a/client/src/app/+my-library/my-auto-tag-policies/my-auto-tag-policies.component.html b/client/src/app/+my-library/my-auto-tag-policies/my-auto-tag-policies.component.htmlindex a0c1ba38bb..f5489e499d 100644--- a/client/src/app/+my-library/my-auto-tag-policies/my-auto-tag-policies.component.html+++ b/client/src/app/+my-library/my-auto-tag-policies/my-auto-tag-policies.component.html@@ -3,7 +3,7 @@@for (tag of tags; track tag; let i = $index) {<div class="form-group ms-3"><my-peertube-checkbox- [inputName]="'tag-' + i" [(ngModel)]="tag.review" [labelText]="getLabelText(tag)"+ [inputName]="'tag-' + i" [(ngModel)]="tag.review" [labelInnerHTML]="getLabelText(tag)"(ngModelChange)="updatePolicies()"></my-peertube-checkbox></div>diff --git a/client/src/app/+my-library/my-auto-tag-policies/my-auto-tag-policies.component.ts b/client/src/app/+my-library/my-auto-tag-policies/my-auto-tag-policies.component.tsindex 53ca256a41..21490269b0 100644--- a/client/src/app/+my-library/my-auto-tag-policies/my-auto-tag-policies.component.ts+++ b/client/src/app/+my-library/my-auto-tag-policies/my-auto-tag-policies.component.ts@@ -1,6 +1,6 @@import { Component, OnInit, inject } from '@angular/core'import { FormsModule } from '@angular/forms'-import { AuthService, Notifier } from '@app/core'+import { AuthService, HtmlRendererService, Notifier } from '@app/core'import { PeertubeCheckboxComponent } from '@app/shared/shared-forms/peertube-checkbox.component'import { AutomaticTagAvailableType } from '@peertube/peertube-models'@@ -18,6 +18,7 @@ export class MyAutoTagPoliciesComponent implements OnInit {private authService = inject(AuthService)private autoTagsService = inject(AutomaticTagService)private notifier = inject(Notifier)+ private html = inject(HtmlRendererService)tags: { name: string, review: boolean, type: AutomaticTagAvailableType }[] = []@@ -26,11 +27,11 @@ export class MyAutoTagPoliciesComponent implements OnInit {}getLabelText (tag: { name: string, type: AutomaticTagAvailableType }) {- if (tag.name === 'external-link') {- return $localize`That contain an external link`- }+ const text = tag.name === 'external-link'+ ? $localize`That contain <strong>an external link</strong>`+ : $localize`That contain any word from your <strong>${tag.name}</strong> watched word list`- return $localize`That contain any word from your "${tag.name}" watched word list`+ return this.html.toSimpleSafeHtml(text)}updatePolicies () {diff --git a/client/src/app/+my-library/my-video-space.component.ts b/client/src/app/+my-library/my-video-space.component.tsindex 7e8d1c3e08..3b435314e1 100644--- a/client/src/app/+my-library/my-video-space.component.ts+++ b/client/src/app/+my-library/my-video-space.component.ts@@ -44,6 +44,10 @@ export class MyVideoSpaceComponent implements OnInit {label: $localize`Watched words`,routerLink: '/my-library/watched-words/list'},+ {+ label: $localize`Watched words subscriptions`,+ routerLink: '/my-library/watched-words/subscriptions'+ },{label: $localize`Auto tag policies`,routerLink: '/my-library/auto-tag-policies'diff --git a/client/src/app/+my-library/my-watched-words-list/my-watched-words-list.component.html b/client/src/app/+my-library/my-watched-words-list/my-watched-words-list.component.htmlindex 46bc8b53b1..21df0e99ff 100644--- a/client/src/app/+my-library/my-watched-words-list/my-watched-words-list.component.html+++ b/client/src/app/+my-library/my-watched-words-list/my-watched-words-list.component.html@@ -1,5 +1,7 @@-<em class="d-block" i18n>Comments that contain any of the watched words are automatically tagged with the name of the list.</em>-<em class="d-block mb-3" i18n>These automatic tags can be used to filter comments or <a routerLink="/my-account/auto-tag-policies">automatically block</a> them.</em>+<div class="callout callout-primary mb-4">+ <div i18n>Comments that contain any of the watched words are automatically tagged with the name of the list.</div>+ <div i18n>These automatic tags can be used to filter comments or <a routerLink="/my-account/auto-tag-policies">automatically block</a> them.</div>+</div><my-watched-words-list-admin-owner mode="user"></my-watched-words-list-admin-owner>diff --git a/client/src/app/+my-library/my-watched-words-subscriptions/my-watched-words-subscriptions.component.html b/client/src/app/+my-library/my-watched-words-subscriptions/my-watched-words-subscriptions.component.htmlnew file mode 100644index 0000000000..111d1f50a6--- /dev/null+++ b/client/src/app/+my-library/my-watched-words-subscriptions/my-watched-words-subscriptions.component.html@@ -0,0 +1,5 @@+<div class="callout callout-primary mb-4" i18n>+ Subscribe to remote watched words lists to automatically keep them synchronized.+</div>++<my-watched-words-subscriptions-admin-owner mode="user"></my-watched-words-subscriptions-admin-owner>diff --git a/client/src/app/+my-library/my-watched-words-subscriptions/my-watched-words-subscriptions.component.ts b/client/src/app/+my-library/my-watched-words-subscriptions/my-watched-words-subscriptions.component.tsnew file mode 100644index 0000000000..1c17736d28--- /dev/null+++ b/client/src/app/+my-library/my-watched-words-subscriptions/my-watched-words-subscriptions.component.ts@@ -0,0 +1,13 @@+import { Component } from '@angular/core'+import { WatchedWordsSubscriptionService } from '@app/shared/shared-watched-words/watched-words-subscription.service'+import {+ WatchedWordsSubscriptionsAdminOwnerComponent+} from '@app/shared/shared-watched-words/watched-words-subscriptions-admin-owner.component'++@Component({+ templateUrl: './my-watched-words-subscriptions.component.html',+ imports: [ WatchedWordsSubscriptionsAdminOwnerComponent ],+ providers: [ WatchedWordsSubscriptionService ]+})+export class MyWatchedWordsSubscriptionsComponent {+}diff --git a/client/src/app/+my-library/routes.ts b/client/src/app/+my-library/routes.tsindex 03ee7a0460..e4fcb20ff1 100644--- a/client/src/app/+my-library/routes.ts+++ b/client/src/app/+my-library/routes.ts@@ -28,6 +28,7 @@ import { MyVideoPlaylistsComponent } from './my-video-playlists/my-video-playlisimport { MyVideoSpaceComponent } from './my-video-space.component'import { MyVideosComponent } from './my-videos/my-videos.component'import { MyWatchedWordsListComponent } from './my-watched-words-list/my-watched-words-list.component'+import { MyWatchedWordsSubscriptionsComponent } from './my-watched-words-subscriptions/my-watched-words-subscriptions.component'const commonConfig = {path: '',@@ -75,6 +76,16 @@ const videoSpaceRoutes = [}},+ {+ path: 'watched-words/subscriptions',+ component: MyWatchedWordsSubscriptionsComponent,+ data: {+ meta: {+ title: $localize`Your watched words subscriptions`+ }+ }+ },+{path: 'auto-tag-policies',component: MyAutoTagPoliciesComponent,diff --git a/client/src/app/shared/shared-watched-words/watched-words-list-admin-owner.component.html b/client/src/app/shared/shared-watched-words/watched-words-list-admin-owner.component.htmlindex 44f407d48b..45922781c0 100644--- a/client/src/app/shared/shared-watched-words/watched-words-list-admin-owner.component.html+++ b/client/src/app/shared/shared-watched-words/watched-words-list-admin-owner.component.html@@ -24,6 +24,12 @@<ng-template #tableCells let-list><td>{{ list.listName }}++ @if (list.subscriptionUrl) {+ <span i18n-title title="Imported from a watched words subscription" class="ms-1 subscription-wrapper">+ <my-global-icon iconName="globe"></my-global-icon>+ </span>+ }</td><td i18n>diff --git a/client/src/app/shared/shared-watched-words/watched-words-list-admin-owner.component.scss b/client/src/app/shared/shared-watched-words/watched-words-list-admin-owner.component.scssnew file mode 100644index 0000000000..3915ba260d--- /dev/null+++ b/client/src/app/shared/shared-watched-words/watched-words-list-admin-owner.component.scss@@ -0,0 +1,8 @@+@use "_variables" as *;+@use "_mixins" as *;++.subscription-wrapper my-global-icon {+ color: pvar(--fg-200);++ @include global-icon-size(16px);+}diff --git a/client/src/app/shared/shared-watched-words/watched-words-list-admin-owner.component.ts b/client/src/app/shared/shared-watched-words/watched-words-list-admin-owner.component.tsindex 2bee3635e1..023de25a05 100644--- a/client/src/app/shared/shared-watched-words/watched-words-list-admin-owner.component.ts+++ b/client/src/app/shared/shared-watched-words/watched-words-list-admin-owner.component.ts@@ -16,6 +16,7 @@ import { WatchedWordsListService } from './watched-words-list.service'@Component({selector: 'my-watched-words-list-admin-owner',templateUrl: './watched-words-list-admin-owner.component.html',+ styleUrls: [ './watched-words-list-admin-owner.component.scss' ],imports: [GlobalIconComponent,ActionDropdownComponent,@@ -90,7 +91,7 @@ export class WatchedWordsListAdminOwnerComponent {}openCreateOrUpdateList (list?: WatchedWordsList) {- this.saveModal().show(list)+ this.saveModal().show({ mode: this.mode(), list })}onListAddedOrUpdated () {diff --git a/client/src/app/shared/shared-watched-words/watched-words-list-save-modal.component.html b/client/src/app/shared/shared-watched-words/watched-words-list-save-modal.component.htmlindex b6d3e4425c..3406f0a790 100644--- a/client/src/app/shared/shared-watched-words/watched-words-list-save-modal.component.html+++ b/client/src/app/shared/shared-watched-words/watched-words-list-save-modal.component.html@@ -9,6 +9,14 @@</div><div class="modal-body">+ <div class="callout callout-primary mb-3">+ @if (mode === 'user') {+ <div i18n>The auto tags on existing comments will be updated based on the words in this list.</div>+ } @else if (mode === 'admin') {+ <div i18n>The auto tags on existing comments and videos will be updated based on the words in this list.</div>+ }+ </div>+<div class="form-group"><label i18n for="listName">List name</label>diff --git a/client/src/app/shared/shared-watched-words/watched-words-list-save-modal.component.ts b/client/src/app/shared/shared-watched-words/watched-words-list-save-modal.component.tsindex 1a795d1c0f..131ff9aec1 100644--- a/client/src/app/shared/shared-watched-words/watched-words-list-save-modal.component.ts+++ b/client/src/app/shared/shared-watched-words/watched-words-list-save-modal.component.ts@@ -23,6 +23,8 @@ export class WatchedWordsListSaveModalComponent extends FormReactive implementsprivate notifier = inject(Notifier)private watchedWordsService = inject(WatchedWordsListService)+ mode: 'admin' | 'user'+readonly accountName = input.required<string>()readonly listAddedOrUpdated = output()@@ -39,15 +41,19 @@ export class WatchedWordsListSaveModalComponent extends FormReactive implements})}- show (list?: WatchedWordsList) {- this.listToUpdate = list+ show (options: {+ mode: 'admin' | 'user'+ list?: WatchedWordsList+ }) {+ this.mode = options.mode+ this.listToUpdate = options.listthis.openedModal = this.modalService.open(this.modal(), { centered: true, keyboard: false })- if (list) {+ if (this.listToUpdate) {this.form.patchValue({- listName: list.listName,- words: list.words.join('\n')+ listName: this.listToUpdate.listName,+ words: this.listToUpdate.words.join('\n')})}}diff --git a/client/src/app/shared/shared-watched-words/watched-words-subscription.service.ts b/client/src/app/shared/shared-watched-words/watched-words-subscription.service.tsnew file mode 100644index 0000000000..5b5228aca9--- /dev/null+++ b/client/src/app/shared/shared-watched-words/watched-words-subscription.service.ts@@ -0,0 +1,58 @@+import { HttpClient, HttpParams } from '@angular/common/http'+import { Injectable, inject } from '@angular/core'+import { RestExtractor, RestPagination, RestService } from '@app/core'+import { ResultList, WatchedWordsSubscription } from '@peertube/peertube-models'+import { SortMeta } from 'primeng/api'+import { catchError } from 'rxjs/operators'+import { environment } from '../../../environments/environment'++@Injectable()+export class WatchedWordsSubscriptionService {+ private authHttp = inject(HttpClient)+ private restExtractor = inject(RestExtractor)+ private restService = inject(RestService)++ private static BASE_WATCHED_WORDS_URL = environment.apiUrl + '/api/v1/watched-words/'++ listSubscriptions (options: {+ accountName?: string+ pagination: RestPagination+ sort: SortMeta+ search?: string+ }) {+ const { pagination, sort, search } = options+ const url = this.buildSubscriptionsPath(options)++ let params = new HttpParams()+ params = this.restService.addRestGetParams(params, pagination, sort)++ if (search) params = params.append('search', search)++ return this.authHttp.get<ResultList<WatchedWordsSubscription>>(url, { params })+ .pipe(catchError(err => this.restExtractor.handleError(err)))+ }++ addSubscription (options: {+ accountName?: string+ url: string+ }) {+ return this.authHttp.post<WatchedWordsSubscription>(this.buildSubscriptionsPath(options), { url: options.url })+ .pipe(catchError(err => this.restExtractor.handleError(err)))+ }++ deleteSubscription (options: {+ accountName?: string+ id: number+ }) {+ return this.authHttp.delete(this.buildSubscriptionsPath(options) + '/' + options.id)+ .pipe(catchError(err => this.restExtractor.handleError(err)))+ }++ private buildSubscriptionsPath (options: { accountName?: string }) {+ const suffixPath = options.accountName+ ? '/accounts/' + options.accountName + '/subscriptions'+ : '/server/subscriptions'++ return WatchedWordsSubscriptionService.BASE_WATCHED_WORDS_URL + suffixPath+ }+}diff --git a/client/src/app/shared/shared-watched-words/watched-words-subscriptions-admin-owner.component.html b/client/src/app/shared/shared-watched-words/watched-words-subscriptions-admin-owner.component.htmlnew file mode 100644index 0000000000..6bf2ba2676--- /dev/null+++ b/client/src/app/shared/shared-watched-words/watched-words-subscriptions-admin-owner.component.html@@ -0,0 +1,103 @@+<my-table+ #table+ key="WatchedWordsSubscriptionsAdminOwnerComponent"+ [defaultColumns]="columns"+ defaultSort="createdAt"+ i18n-paginatorText+ paginatorText="subscriptions per page"+ [dataLoader]="dataLoader"+ columnConfig="false"+ searchInput="true"+>+ <ng-template #totalTitle let-totalRecords>+ <ng-container i18n>{ totalRecords, plural, =0 {No subscription} =1 {1 subscription} other {{{ totalRecords | myNumberFormatter }} subscriptions}}</ng-container>+ </ng-template>++ <ng-template #captionRight>+ <button type="button" class="peertube-create-button" (click)="openAddModal()">+ <my-global-icon iconName="add" aria-hidden="true"></my-global-icon>+ <ng-container i18n>Add a subscription</ng-container>+ </button>+ </ng-template>++ <ng-template #tableCells let-subscription>+ <td>{{ subscription.name }}</td>++ <td>+ <a [href]="subscription.url" target="_blank" rel="noopener noreferrer">+ {{ subscription.url }}+ <my-global-icon iconName="external-link"></my-global-icon>+ </a>+ </td>++ <td>+ <span [ngClass]="getStateClass(subscription.state.id)">+ {{ subscription.state.label }}+ </span>+ </td>++ <td i18n>+ {{ subscription.importedWordsCount }} words+ </td>++ <td>{{ subscription.lastSyncAt ? (subscription.lastSyncAt | ptDate: 'short') : '-' }}</td>++ <td>{{ subscription.createdAt | ptDate: 'short' }}</td>+ </ng-template>++ <ng-template #actionCell let-subscription>+ <button class="peertube-button secondary-button" (click)="removeSubscription(subscription)" i18n>Delete</button>+ </ng-template>++ <ng-template #noResults let-search="search" let-hasFilters="hasFilters">+ @if (search || hasFilters) {+ <ng-container i18n>No subscription found matching current filters.</ng-container>+ } @else {+ <ng-container i18n>No subscription found.</ng-container>+ }+ </ng-template>+</my-table>++<ng-template #addModal let-hide="close">+ <div class="modal-container">+ <div class="modal-header">+ <h4 class="modal-title" i18n>Add a watched words subscription</h4>++ <button class="border-0 p-0" (click)="hide()" i18n-title title="Close">+ <my-global-icon iconName="cross"></my-global-icon>+ </button>+ </div>++ <div class="modal-body">+ <div class="callout callout-primary mb-3" i18n>+ Enter the URL of the watched words list to subscribe to. The remote list will be fetched and synchronized periodically.+ </div>++ <div class="form-group">+ <label i18n for="subscriptionUrl">Watched words URL</label>+ <input+ type="url"+ id="subscriptionUrl"+ class="form-control"+ [formField]="addSubscriptionForm['url']"+ i18n-placeholder+ placeholder="https://example.com/watched-words.json"+ >++ <my-form-error [field]="addSubscriptionForm.url()"></my-form-error>+ </div>+ </div>++ <div class="modal-footer">+ <button class="peertube-button secondary-button" (click)="hide()" i18n>Cancel</button>++ <my-button+ theme="primary"+ [loading]="addingSubscription"+ [disabled]="!addSubscriptionForm().valid() || addingSubscription"+ (click)="submitAddModal()"+ i18n+ >Subscribe</my-button>+ </div>+ </div>+</ng-template>diff --git a/client/src/app/shared/shared-watched-words/watched-words-subscriptions-admin-owner.component.ts b/client/src/app/shared/shared-watched-words/watched-words-subscriptions-admin-owner.component.tsnew file mode 100644index 0000000000..9b6a58f3c5--- /dev/null+++ b/client/src/app/shared/shared-watched-words/watched-words-subscriptions-admin-owner.component.ts@@ -0,0 +1,178 @@+import { CommonModule } from '@angular/common'+import { Component, ElementRef, inject, input, signal, viewChild } from '@angular/core'+import { form, FormField, pattern, required } from '@angular/forms/signals'+import { AuthService, ConfirmService, Notifier } from '@app/core'+import { urlPattern } from '@app/shared/form-validators/common-validators'+import { FormErrorComponent } from '@app/shared/shared-forms/form-error.component'+import { GlobalIconComponent } from '@app/shared/shared-icons/global-icon.component'+import { ButtonComponent } from '@app/shared/shared-main/buttons/button.component'+import { PTDatePipe } from '@app/shared/shared-main/common/date.pipe'+import { NumberFormatterPipe } from '@app/shared/shared-main/common/number-formatter.pipe'+import { getStateBadgeClasses } from '@app/shared/shared-tables/state-badge'+import { DataLoaderOptionsBase, TableColumnInfo, TableComponent } from '@app/shared/shared-tables/table.component'+import { NgbModal, NgbModalRef } from '@ng-bootstrap/ng-bootstrap'+import { StreamSyncStateType, WatchedWordsSubscription } from '@peertube/peertube-models'+import { first, switchMap } from 'rxjs'+import { WatchedWordsSubscriptionService } from './watched-words-subscription.service'++type AddSubscriptionFormModel = {+ url: string+}++@Component({+ selector: 'my-watched-words-subscriptions-admin-owner',+ templateUrl: './watched-words-subscriptions-admin-owner.component.html',+ imports: [+ CommonModule,+ FormErrorComponent,+ TableComponent,+ NumberFormatterPipe,+ PTDatePipe,+ GlobalIconComponent,+ FormField,+ ButtonComponent+ ]+})+export class WatchedWordsSubscriptionsAdminOwnerComponent {+ private auth = inject(AuthService)+ private notifier = inject(Notifier)+ private confirmService = inject(ConfirmService)+ private modalService = inject(NgbModal)+ private watchedWordsSubscriptionService = inject(WatchedWordsSubscriptionService)++ readonly mode = input.required<'user' | 'admin'>()++ readonly table = viewChild<TableComponent<WatchedWordsSubscription>>('table')+ readonly addModal = viewChild<ElementRef>('addModal')++ readonly addSubscriptionModel = signal<AddSubscriptionFormModel>({ url: '' })+ readonly addSubscriptionForm = form(this.addSubscriptionModel, f => {+ required(f.url, { message: $localize`Watched words URL is required.` })+ pattern(f.url, urlPattern, { message: $localize`Please enter a valid URL.` })+ })+ addingSubscription = false++ private addModalOpened: NgbModalRef++ columns: TableColumnInfo<string>[] = [+ {+ id: 'name',+ label: $localize`List name`,+ selected: true,+ sortable: true+ },+ {+ id: 'url',+ label: $localize`URL`,+ selected: true,+ sortable: false+ },+ {+ id: 'syncStatus',+ label: $localize`Status`,+ selected: true,+ sortable: false+ },+ {+ id: 'importedWordsCount',+ label: $localize`Words`,+ selected: true,+ sortable: false+ },+ {+ id: 'lastSyncAt',+ label: $localize`Last synced`,+ selected: true,+ sortable: true+ },+ {+ id: 'createdAt',+ label: $localize`Subscribed at`,+ selected: true,+ sortable: true+ }+ ]++ dataLoader: typeof this._dataLoader++ constructor () {+ this.dataLoader = this._dataLoader.bind(this)+ }++ openAddModal () {+ this.addSubscriptionModel.set({ url: '' })+ this.addSubscriptionForm().reset()++ this.addModalOpened = this.modalService.open(this.addModal(), { centered: true })+ }++ submitAddModal () {+ const url = this.addSubscriptionModel().url.trim()++ this.addingSubscription = true++ this.watchedWordsSubscriptionService.addSubscription({ url, accountName: this.getAccountNameParam() })+ .subscribe({+ next: () => {+ this.notifier.success($localize`Subscription to ${url} added.`)++ this.addModalOpened.close()+ this.table().loadData()+ this.addingSubscription = false+ },++ error: err => {+ this.notifier.handleError(err)+ this.addingSubscription = false+ }+ })+ }++ async removeSubscription (subscription: WatchedWordsSubscription) {+ const message = this.mode() === 'admin'+ ? $localize`Deleting this subscription will remove <strong>${subscription.name}</strong> auto tags assigned to comments and videos.`+ : $localize`Deleting this subscription will remove <strong>${subscription.name}</strong> auto tags assigned to comments.`++ const res = await this.confirmService.confirm(message, $localize`Delete subscription`)+ if (res === false) return++ this.watchedWordsSubscriptionService.deleteSubscription({+ id: subscription.id,+ accountName: this.getAccountNameParam()+ }).subscribe({+ next: () => {+ this.notifier.success($localize`Subscription to <strong>${subscription.name}</strong> deleted.`)+ this.table().loadData()+ },++ error: err => this.notifier.handleError(err)+ })+ }++ getStateClass (stateId: StreamSyncStateType) {+ return getStateBadgeClasses(stateId)+ }++ private getAccountNameParam () {+ if (this.mode() === 'admin') return undefined++ return this.auth.getUser().account.name+ }++ private _dataLoader (options: DataLoaderOptionsBase) {+ const { pagination, sort, search } = options++ return this.auth.userInformationLoaded+ .pipe(+ first(),+ switchMap(() =>+ this.watchedWordsSubscriptionService.listSubscriptions({+ pagination,+ sort,+ search,+ accountName: this.getAccountNameParam()+ })+ )+ )+ }+}diff --git a/packages/models/src/moderation/index.ts b/packages/models/src/moderation/index.tsindex 671119611c..2fc83ddd78 100644--- a/packages/models/src/moderation/index.ts+++ b/packages/models/src/moderation/index.ts@@ -10,3 +10,5 @@ export * from './blocklist-log.model.js'export * from './blocklist-subscription.model.js'export * from './server-block.model.js'export * from './watched-words-list.model.js'+export * from './watched-words-subscription-actions.model.js'+export * from './watched-words-subscription.model.js'diff --git a/packages/models/src/moderation/watched-words-list.model.ts b/packages/models/src/moderation/watched-words-list.model.tsindex c0312e86d0..1ca2e52d92 100644--- a/packages/models/src/moderation/watched-words-list.model.ts+++ b/packages/models/src/moderation/watched-words-list.model.ts@@ -3,6 +3,7 @@ export interface WatchedWordsList {listName: stringwords: string[]+ subscriptionUrl: string | nullupdatedAt: Date | stringcreatedAt: Date | stringdiff --git a/packages/models/src/moderation/watched-words-subscription-actions.model.ts b/packages/models/src/moderation/watched-words-subscription-actions.model.tsnew file mode 100644index 0000000000..d9a51ee07a--- /dev/null+++ b/packages/models/src/moderation/watched-words-subscription-actions.model.ts@@ -0,0 +1,12 @@+export type WatchedWordsSubscriptionActionType = 'add' | 'remove'++export interface WatchedWordsSubscriptionAction {+ type: WatchedWordsSubscriptionActionType+ word: string+ createdAt: string+}++export interface WatchedWordsSubscriptionActions {+ name: string+ actions: WatchedWordsSubscriptionAction[]+}diff --git a/packages/models/src/moderation/watched-words-subscription.model.ts b/packages/models/src/moderation/watched-words-subscription.model.tsnew file mode 100644index 0000000000..4003d02fcd--- /dev/null+++ b/packages/models/src/moderation/watched-words-subscription.model.ts@@ -0,0 +1,16 @@+import { ConstantLabel } from '../common/constant-label.model.js'+import { StreamSyncStateType } from '../common/stream-sync-state.enum.js'++export interface WatchedWordsSubscription {+ id: number+ name: string+ url: string++ importedWordsCount: number++ lastSyncAt: Date | string+ state: ConstantLabel<StreamSyncStateType>++ createdAt: Date | string+ updatedAt: Date | string+}diff --git a/packages/models/src/server/job.model.ts b/packages/models/src/server/job.model.tsindex f745397074..cb7f627ef9 100644--- a/packages/models/src/server/job.model.ts+++ b/packages/models/src/server/job.model.ts@@ -15,13 +15,17 @@ export type JobType =| 'activitypub-refresher'| 'actor-keys'| 'after-video-channel-import'+ | 'build-automatic-tags'+ | 'create-user-export'| 'email'| 'federate-video'- | 'transcoding-job-builder'+ | 'generate-video-storyboard'+ | 'import-user-archive'| 'manage-video-torrent'- | 'move-to-object-storage'| 'move-to-file-system'+ | 'move-to-object-storage'| 'notify'+ | 'transcoding-job-builder'| 'video-channel-import'| 'video-file-import'| 'video-import'@@ -29,11 +33,8 @@ export type JobType =| 'video-redundancy'| 'video-studio-edition'| 'video-transcoding'- | 'videos-stats'- | 'generate-video-storyboard'- | 'create-user-export'- | 'import-user-archive'| 'video-transcription'+ | 'videos-stats'export interface Job {id: number | string@@ -359,3 +360,11 @@ export interface ImportUserArchivePayload {export interface VideoTranscriptionPayload {videoUUID: string}++// ---------------------------------------------------------------------------++export interface BuildAutomaticTagsPayload {+ accountId: number+ ofComments: boolean+ ofVideos: boolean+}diff --git a/packages/server-commands/src/moderation/watched-words-command.ts b/packages/server-commands/src/moderation/watched-words-command.tsindex 92566f8e57..f793e31a1a 100644--- a/packages/server-commands/src/moderation/watched-words-command.ts+++ b/packages/server-commands/src/moderation/watched-words-command.ts@@ -1,20 +1,18 @@import { pick } from '@peertube/peertube-core-utils'-import {- HttpStatusCode,- ResultList, WatchedWordsList-} from '@peertube/peertube-models'+import { HttpStatusCode, ResultList, WatchedWordsList, WatchedWordsSubscription } from '@peertube/peertube-models'import { unwrapBody } from '../index.js'import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js'export class WatchedWordsCommand extends AbstractCommand {+ listWordsLists (+ options: OverrideCommandOptions & {+ start?: number+ count?: number+ sort?: string- listWordsLists (options: OverrideCommandOptions & {- start?: number- count?: number- sort?: string-- accountName?: string- }) {+ accountName?: string+ }+ ) {const query = {sort: '-createdAt',@@ -31,11 +29,13 @@ export class WatchedWordsCommand extends AbstractCommand {})}- createList (options: OverrideCommandOptions & {- listName: string- words: string[]- accountName?: string- }) {+ createList (+ options: OverrideCommandOptions & {+ listName: string+ words: string[]+ accountName?: string+ }+ ) {const body = pick(options, [ 'listName', 'words' ])return unwrapBody<{ watchedWordsList: { id: number } }>(this.postBodyRequest({@@ -48,12 +48,14 @@ export class WatchedWordsCommand extends AbstractCommand {}))}- updateList (options: OverrideCommandOptions & {- listId: number- accountName?: string- listName?: string- words?: string[]- }) {+ updateList (+ options: OverrideCommandOptions & {+ listId: number+ accountName?: string+ listName?: string+ words?: string[]+ }+ ) {const body = pick(options, [ 'listName', 'words' ])return this.putBodyRequest({@@ -66,10 +68,12 @@ export class WatchedWordsCommand extends AbstractCommand {})}- deleteList (options: OverrideCommandOptions & {- listId: number- accountName?: string- }) {+ deleteList (+ options: OverrideCommandOptions & {+ listId: number+ accountName?: string+ }+ ) {return this.deleteRequest({...options,@@ -79,9 +83,71 @@ export class WatchedWordsCommand extends AbstractCommand {})}+ listWordsSubscriptions (options: OverrideCommandOptions & {+ start?: number+ count?: number+ sort?: string+ search?: string+ accountName?: string+ } = {}) {+ const query = {+ sort: '-createdAt',++ ...pick(options, [ 'start', 'count', 'sort', 'search' ])+ }++ return this.getRequestBody<ResultList<WatchedWordsSubscription>>({+ ...options,++ path: this.buildSubscriptionAPIBasePath(options.accountName),+ query,+ implicitToken: true,+ defaultExpectedStatus: HttpStatusCode.OK_200+ })+ }++ createSubscription (+ options: OverrideCommandOptions & {+ subscriptionUrl: string+ accountName?: string+ }+ ) {+ return unwrapBody<WatchedWordsSubscription>(this.postBodyRequest({+ ...options,++ path: this.buildSubscriptionAPIBasePath(options.accountName),+ fields: {+ url: options.subscriptionUrl+ },+ implicitToken: true,+ defaultExpectedStatus: HttpStatusCode.OK_200+ }))+ }++ deleteSubscription (+ options: OverrideCommandOptions & {+ id: number | string+ accountName?: string+ }+ ) {+ return this.deleteRequest({+ ...options,++ path: this.buildSubscriptionAPIBasePath(options.accountName) + '/' + options.id,+ implicitToken: true,+ defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204+ })+ }+private buildAPIBasePath (accountName?: string) {return accountName? '/api/v1/watched-words/accounts/' + accountName + '/lists': '/api/v1/watched-words/server/lists'}++ private buildSubscriptionAPIBasePath (accountName?: string) {+ return accountName+ ? '/api/v1/watched-words/accounts/' + accountName + '/subscriptions'+ : '/api/v1/watched-words/server/subscriptions'+ }}diff --git a/packages/tests/src/api/check-params/watched-words-subscriptions.ts b/packages/tests/src/api/check-params/watched-words-subscriptions.tsnew file mode 100644index 0000000000..7b8771ee41--- /dev/null+++ b/packages/tests/src/api/check-params/watched-words-subscriptions.ts@@ -0,0 +1,276 @@+/* oxlint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */++import { wait } from '@peertube/peertube-core-utils'+import { HttpStatusCode, UserRole } from '@peertube/peertube-models'+import {+ PeerTubeServer,+ WatchedWordsCommand,+ cleanupTests,+ createSingleServer,+ makeGetRequest,+ setAccessTokensToServers,+ setDefaultAccountAvatar+} from '@peertube/peertube-server-commands'+import { checkBadCountPagination, checkBadSort, checkBadStartPagination } from '@tests/shared/checks.js'+import { MockWatchedWords } from '@tests/shared/mock-servers/mock-watched-words.js'++describe('Test watched words subscriptions API validators', function () {+ let server: PeerTubeServer+ let mockServer: MockWatchedWords+ let mockPort: number++ let userToken: string+ let userToken2: string+ let moderatorToken: string++ let command: WatchedWordsCommand++ let accountSubscriptionId: number+ let serverSubscriptionId: number++ before(async function () {+ this.timeout(120000)++ server = await createSingleServer(1)+ await setAccessTokensToServers([ server ])+ await setDefaultAccountAvatar([ server ])++ mockServer = new MockWatchedWords()+ mockServer.setActions([+ {+ type: 'add',+ word: 'word 1',+ createdAt: new Date().toISOString()+ }+ ])+ mockPort = await mockServer.initialize()++ userToken = await server.users.generateUserAndToken('user1')+ userToken2 = await server.users.generateUserAndToken('user2')+ moderatorToken = await server.users.generateUserAndToken('moderator', UserRole.MODERATOR)++ command = server.watchedWordsLists++ {+ const subscription = await command.createSubscription({+ accountName: 'user1',+ token: userToken,+ subscriptionUrl: `http://127.0.0.1:${mockPort}/list.json`+ })++ accountSubscriptionId = subscription.id+ }++ {+ const subscription = await command.createSubscription({+ token: moderatorToken,+ subscriptionUrl: `http://127.0.0.1:${mockPort}/list-3`+ })++ serverSubscriptionId = subscription.id+ }+ })++ describe('Account specific watched words subscriptions', function () {+ it('Should fail with an unknown account when listing subscriptions', async function () {+ await command.listWordsSubscriptions({ accountName: 'unknown', expectedStatus: HttpStatusCode.NOT_FOUND_404 })+ })++ it('Should fail with an unknown account when adding subscriptions', async function () {+ await command.createSubscription({+ token: userToken,+ accountName: 'unknown',+ subscriptionUrl: `http://127.0.0.1:${mockPort}/list.json`,+ expectedStatus: HttpStatusCode.NOT_FOUND_404+ })+ })++ it('Should fail with an unknown account when deleting subscriptions', async function () {+ await command.deleteSubscription({+ token: userToken,+ accountName: 'unknown',+ id: accountSubscriptionId,+ expectedStatus: HttpStatusCode.NOT_FOUND_404+ })+ })+ })++ describe('Account & server watched words subscriptions', function () {+ describe('When listing subscriptions', function () {+ const paths = () => [+ { path: '/api/v1/watched-words/accounts/user1/subscriptions', token: userToken },+ { path: '/api/v1/watched-words/server/subscriptions', token: moderatorToken }+ ]++ it('Should fail with an unauthenticated user', async function () {+ for (const { path } of paths()) {+ await makeGetRequest({+ url: server.url,+ path,+ expectedStatus: HttpStatusCode.UNAUTHORIZED_401+ })+ }+ })++ it('Should fail with the wrong token', async function () {+ for (const { path } of paths()) {+ await makeGetRequest({+ url: server.url,+ token: userToken2,+ path,+ expectedStatus: HttpStatusCode.FORBIDDEN_403+ })+ }+ })++ it('Should fail with a bad start/count pagination or incorrect sort', async function () {+ for (const { path, token } of paths()) {+ await checkBadStartPagination(server.url, path, token)+ await checkBadCountPagination(server.url, path, token)+ await checkBadSort(server.url, path, token)+ }+ })++ it('Should succeed with the right token', async function () {+ for (const { path, token } of paths()) {+ await makeGetRequest({ url: server.url, token, path, expectedStatus: HttpStatusCode.OK_200 })+ }+ })+ })++ describe('When adding subscriptions', function () {+ const baseParams = () => [+ {+ token: userToken,+ accountName: 'user1',+ subscriptionUrl: `http://127.0.0.1:${mockPort}/list-3`+ },+ {+ token: moderatorToken,+ subscriptionUrl: `http://127.0.0.1:${mockPort}/list.json`+ }+ ]++ it('Should fail with an unauthenticated user', async function () {+ for (const baseParam of baseParams()) {+ await command.createSubscription({ ...baseParam, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })+ }+ })++ it('Should fail with the wrong token', async function () {+ for (const baseParam of baseParams()) {+ await command.createSubscription({ ...baseParam, token: userToken2, expectedStatus: HttpStatusCode.FORBIDDEN_403 })+ }+ })++ it('Should fail with an invalid URL', async function () {+ for (const baseParam of baseParams()) {+ await command.createSubscription({ ...baseParam, subscriptionUrl: 'invalid', expectedStatus: HttpStatusCode.BAD_REQUEST_400 })+ }+ })++ it('Should fail with an URL that does not have the right format', async function () {+ for (const baseParam of baseParams()) {+ await command.createSubscription({+ ...baseParam,+ subscriptionUrl: `http://127.0.0.1:${mockPort}/invalid-watched-words-1`,+ expectedStatus: HttpStatusCode.BAD_REQUEST_400+ })++ await command.createSubscription({+ ...baseParam,+ subscriptionUrl: `http://127.0.0.1:${mockPort}/invalid-watched-words-2`,+ expectedStatus: HttpStatusCode.BAD_REQUEST_400+ })+ }+ })++ it('Should succeed with the correct parameters', async function () {+ for (const baseParam of baseParams()) {+ await command.createSubscription(baseParam)+ }+ })++ it('Should fail with an already existing URL', async function () {+ await command.createSubscription({+ token: userToken,+ accountName: 'user1',+ subscriptionUrl: `http://127.0.0.1:${mockPort}/list.json`,+ expectedStatus: HttpStatusCode.CONFLICT_409+ })++ await command.createSubscription({+ token: moderatorToken,+ subscriptionUrl: `http://127.0.0.1:${mockPort}/list-3`,+ expectedStatus: HttpStatusCode.CONFLICT_409+ })+ })++ it('Should fail with an already existing list name', async function () {+ await wait(6000)++ await command.createSubscription({+ token: userToken,+ accountName: 'user1',+ subscriptionUrl: `http://127.0.0.1:${mockPort}/list-copy.json`,+ expectedStatus: HttpStatusCode.CONFLICT_409+ })++ await command.createSubscription({+ token: moderatorToken,+ subscriptionUrl: `http://127.0.0.1:${mockPort}/list-3-copy`,+ expectedStatus: HttpStatusCode.CONFLICT_409+ })+ })+ })++ describe('When deleting subscriptions', function () {+ const baseParams = () => [+ {+ token: userToken,+ accountName: 'user1',+ id: accountSubscriptionId+ },+ {+ token: moderatorToken,+ id: serverSubscriptionId+ }+ ]++ it('Should fail with an unauthenticated user', async function () {+ for (const baseParam of baseParams()) {+ await command.deleteSubscription({ ...baseParam, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })+ }+ })++ it('Should fail with the wrong token', async function () {+ for (const baseParam of baseParams()) {+ await command.deleteSubscription({ ...baseParam, token: userToken2, expectedStatus: HttpStatusCode.FORBIDDEN_403 })+ }+ })++ it('Should fail with an invalid id', async function () {+ for (const baseParam of baseParams()) {+ await command.deleteSubscription({ ...baseParam, id: 'invalid', expectedStatus: HttpStatusCode.BAD_REQUEST_400 })+ }+ })++ it('Should fail with an unknown id', async function () {+ for (const baseParam of baseParams()) {+ await command.deleteSubscription({ ...baseParam, id: 999999, expectedStatus: HttpStatusCode.NOT_FOUND_404 })+ }+ })++ it('Should succeed with the correct parameters', async function () {+ for (const baseParam of baseParams()) {+ await command.deleteSubscription(baseParam)+ }+ })+ })+ })++ after(async function () {+ await cleanupTests([ server ])+ await mockServer.terminate()+ })+})diff --git a/packages/tests/src/api/moderation/automatic-tags.ts b/packages/tests/src/api/moderation/automatic-tags.tsindex 5751e39f98..938833e3b6 100644--- a/packages/tests/src/api/moderation/automatic-tags.ts+++ b/packages/tests/src/api/moderation/automatic-tags.ts@@ -29,7 +29,6 @@ describe('Test automatic tags', function () {await servers[1].config.enableLive({ allowReplay: false })await doubleFollow(servers[0], servers[1])-;({ uuid: videoUUID } = await servers[0].videos.quickUpload({ name: 'video' }))await waitJobs(servers)@@ -183,13 +182,15 @@ describe('Test automatic tags', function () {listName: 'list 3'})+ await waitJobs(servers)+await servers[0].comments.createThread({ videoId: videoUUID, text: 'captain nemo' })await servers[0].comments.createThread({ videoId: videoUUID, text: 'my nautilus 2' })await servers[0].comments.createThread({ videoId: videoUUID, text: 'word 1' })const { data } = await servers[0].comments.listCommentsOnMyVideos()- // Previous comment still have the same automatic tags- expect(data.find(c => c.text === 'my nautilus').automaticTags).to.have.lengthOf(0)+ // Previous comment has been rebuilt using the new watched words list+ expect(data.find(c => c.text === 'my nautilus').automaticTags).to.have.members([ 'list 3' ])expect(data.find(c => c.text === 'captain nemo').automaticTags).to.have.lengthOf(0)expect(data.find(c => c.text === 'my nautilus 2').automaticTags).to.have.members([ 'list 3' ])@@ -406,7 +407,7 @@ describe('Test automatic tags', function () {})it('Should update watched words list and assign auto tag on update', async function () {- const { uuid } = await servers[0].videos.quickUpload({ name: 'hi minnie' })+ await servers[0].videos.quickUpload({ name: 'hi minnie' }){const { data } = await servers[0].videos.listAllForAdmin()@@ -420,42 +421,13 @@ describe('Test automatic tags', function () {listName: 'mickey list v2'})- await servers[0].videos.update({ id: uuid, attributes: { name: 'hi minnie v2' } })+ await waitJobs(servers)const { data } = await servers[0].videos.listAllForAdmin()- expect(data.find(v => v.name === 'hi minnie v2').automaticTags).to.have.members([ 'mickey list v2' ])+ expect(data.find(v => v.name === 'hi minnie').automaticTags).to.have.members([ 'mickey list v2' ])}})- it('Should not update remote video if name/description has not changed', async function () {- await servers[1].videos.update({- id: liveUUID,- attributes: {- channelId: servers[0].store.channel.id,- tags: [ 'super tag' ]- }- })-- await waitJobs(servers)-- const { data } = await servers[0].videos.listAllForAdmin()- expect(data.find(v => v.name === 'live loulou').automaticTags).to.have.members([ 'donald list', 'mickey list' ])- })-- it('Should update remote video if name/description has changed', async function () {- await servers[1].videos.update({- id: liveUUID,- attributes: { name: 'live loulou v2' }- })-- await waitJobs(servers)-- const { data } = await servers[0].videos.listAllForAdmin()- expect(data.find(v => v.name === 'live loulou v2').automaticTags).to.have.members([ 'donald list', 'mickey list v2' ])- })- })-- describe('Searching videos with specific tags', function () {it('Should search in admin videos with specific automatic tags', async function () {{const { total, data } = await servers[0].videos.listAllForAdmin({ autoTagOneOf: [ 'picsou list' ] })@@ -470,9 +442,31 @@ describe('Test automatic tags', function () {expect(total).to.equal(2)expect(data).to.have.lengthOf(2)- expect(data.map(d => d.name)).to.have.members([ 'hi minnie v2', 'live loulou v2' ])+ expect(data.map(d => d.name)).to.have.members([ 'hi minnie', 'live loulou' ])}})++ it('Should update remote video if name/description has changed', async function () {+ await servers[1].videos.update({+ id: liveUUID,+ attributes: { name: 'live v2' }+ })++ await waitJobs(servers)++ const { data } = await servers[0].videos.listAllForAdmin()+ expect(data.find(v => v.name === 'live v2').automaticTags).to.have.members([ 'mickey list v2' ])+ })++ it('Should delete watched words list and automatically delete auto tags', async function () {+ await servers[0].watchedWordsLists.deleteList({ listId: serverListId })+ await waitJobs(servers)++ const { total, data } = await servers[0].videos.listAllForAdmin({ autoTagOneOf: [ 'mickey list v2' ] })+ console.log(data)+ expect(total).to.equal(0)+ expect(data).to.have.lengthOf(0)+ })})})diff --git a/packages/tests/src/api/moderation/watched-words-subscriptions.ts b/packages/tests/src/api/moderation/watched-words-subscriptions.tsnew file mode 100644index 0000000000..e54b96a236--- /dev/null+++ b/packages/tests/src/api/moderation/watched-words-subscriptions.ts@@ -0,0 +1,255 @@+/* oxlint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */++import { wait } from '@peertube/peertube-core-utils'+import { UserRole } from '@peertube/peertube-models'+import {+ cleanupTests,+ createSingleServer,+ PeerTubeServer,+ setAccessTokensToServers,+ setDefaultAccountAvatar,+ waitJobs+} from '@peertube/peertube-server-commands'+import { MockWatchedWords } from '@tests/shared/mock-servers/mock-watched-words.js'+import { expect } from 'chai'++describe('Test watched words subscriptions', function () {+ let server: PeerTubeServer+ let mockServer: MockWatchedWords+ let mockPort: number+ let userToken: string+ let moderatorToken: string++ before(async function () {+ this.timeout(120000)++ server = await createSingleServer(1)+ await setAccessTokensToServers([ server ])+ await setDefaultAccountAvatar([ server ])++ mockServer = new MockWatchedWords()+ mockPort = await mockServer.initialize()++ userToken = await server.users.generateUserAndToken('user1')+ moderatorToken = await server.users.generateUserAndToken('moderator', UserRole.MODERATOR)+ })++ function runTests (mode: 'server' | 'account') {+ let subscriptionId: number+ let accountName: string+ let token: string++ before(() => {+ accountName = mode === 'server'+ ? undefined+ : 'user1'++ token = mode === 'server'+ ? moderatorToken+ : userToken++ mockServer.setName('Remote watched words')+ mockServer.setActions([])+ })++ it('Should list empty subscriptions', async function () {+ const { data, total } = await server.watchedWordsLists.listWordsSubscriptions({ token, accountName })++ expect(total).to.equal(0)+ expect(data).to.have.lengthOf(0)+ })++ it('Should add a subscription', async function () {+ const subscription = await server.watchedWordsLists.createSubscription({+ token,+ accountName,+ subscriptionUrl: `http://127.0.0.1:${mockPort}/list.json`+ })++ expect(subscription.name).to.equal('Remote watched words')+ expect(subscription.url).to.equal(`http://127.0.0.1:${mockPort}/list.json`)+ expect(subscription.importedWordsCount).to.equal(0)++ subscriptionId = subscription.id+ })++ it('Should list subscriptions', async function () {+ const { data, total } = await server.watchedWordsLists.listWordsSubscriptions({ token, accountName })++ expect(total).to.equal(1)+ expect(data).to.have.lengthOf(1)+ expect(data[0].name).to.equal('Remote watched words')+ expect(data[0].lastSyncAt).to.be.null+ expect(data[0].importedWordsCount).to.equal(0)+ })++ it('Should synchronize watched words and rename the imported list', async function () {+ this.timeout(120000)++ const now = Date.now()+ const videoName = `subscription video word2 ${mode}`+ const commentText = `subscription comment word2 ${mode}`++ const { uuid } = await server.videos.quickUpload({ token, name: videoName })+ await server.comments.createThread({ token, videoId: uuid, text: commentText })++ await waitJobs([ server ])++ {+ const { data } = await server.videos.listAllForAdmin()+ expect(data.find(v => v.name === videoName).automaticTags).to.have.lengthOf(0)+ }++ {+ const comments = mode === 'server'+ ? await server.comments.listForAdmin()+ : await server.comments.listCommentsOnMyVideos({ token })++ expect(comments.data.find(c => c.text === commentText).automaticTags).to.have.lengthOf(0)+ }++ mockServer.setActions([+ { type: 'add', word: 'word1', createdAt: new Date(now - 1000).toISOString() },+ { type: 'add', word: 'word2', createdAt: new Date(now - 500).toISOString() }+ ])++ await wait(8000)+ await waitJobs([ server ])++ {+ const { data, total } = await server.watchedWordsLists.listWordsLists({ token, accountName })++ expect(total).to.equal(1)+ expect(data[0].listName).to.equal('Remote watched words')+ expect(data[0].words).to.deep.equal([ 'word1', 'word2' ])+ expect(data[0].subscriptionUrl).to.equal(`http://127.0.0.1:${mockPort}/list.json`)+ }++ {+ const { data } = await server.watchedWordsLists.listWordsSubscriptions({ token, accountName })++ expect(data[0].importedWordsCount).to.equal(2)+ expect(data[0].state.label).to.be.oneOf([ 'Synchronized', 'Processing' ])+ }++ {+ const comments = mode === 'server'+ ? await server.comments.listForAdmin()+ : await server.comments.listCommentsOnMyVideos({ token })++ expect(comments.data.find(c => c.text === commentText).automaticTags).to.have.members([ 'Remote watched words' ])+ }++ {+ const { data } = await server.videos.listAllForAdmin()+ const automaticTags = data.find(v => v.name === videoName).automaticTags++ if (mode === 'server') expect(automaticTags).to.have.members([ 'Remote watched words' ])+ else expect(automaticTags).to.have.lengthOf(0)+ }++ mockServer.setName('Renamed watched words')+ mockServer.setActions([+ { type: 'add', word: 'word1', createdAt: new Date(now - 1000).toISOString() },+ { type: 'add', word: 'word2', createdAt: new Date(now - 500).toISOString() },+ { type: 'remove', word: 'word1', createdAt: new Date(now + 1000).toISOString() },+ { type: 'add', word: 'word3', createdAt: new Date(now + 2000).toISOString() }+ ])++ await wait(8000)+ await waitJobs([ server ])++ const { data } = await server.watchedWordsLists.listWordsLists({ token, accountName })+ expect(data).to.have.lengthOf(1)+ expect(data[0].listName).to.equal('Renamed watched words')+ expect(data[0].words).to.deep.equal([ 'word2', 'word3' ])+ expect(data[0].subscriptionUrl).to.equal(`http://127.0.0.1:${mockPort}/list.json`)++ {+ const comments = mode === 'server'+ ? await server.comments.listForAdmin()+ : await server.comments.listCommentsOnMyVideos({ token })++ expect(comments.data.find(c => c.text === commentText).automaticTags).to.have.members([ 'Renamed watched words' ])+ }++ {+ const { data } = await server.videos.listAllForAdmin()+ const automaticTags = data.find(v => v.name === videoName).automaticTags++ if (mode === 'server') expect(automaticTags).to.have.members([ 'Renamed watched words' ])+ else expect(automaticTags).to.have.lengthOf(0)+ }+ })++ it('Should delete a subscription and remove imported watched words', async function () {+ const unrelatedSubscription = await server.watchedWordsLists.createSubscription({+ token,+ accountName,+ subscriptionUrl: `http://127.0.0.1:${mockPort}/list-2`+ })++ const unrelatedSubscriptionWithActions = await server.watchedWordsLists.createSubscription({+ token,+ accountName,+ subscriptionUrl: `http://127.0.0.1:${mockPort}/list-3`+ })++ await server.watchedWordsLists.createList({+ token,+ accountName,+ listName: 'Manual watched words list',+ words: [ 'manual-word' ]+ })++ await wait(8000)++ await server.watchedWordsLists.deleteSubscription({ token, accountName, id: subscriptionId })++ {+ const { total, data } = await server.watchedWordsLists.listWordsSubscriptions({ token, accountName })++ expect(total).to.equal(2)+ expect(data).to.have.lengthOf(2)++ const subscriptionIds = data.map(subscription => subscription.id)+ const subscriptionNames = data.map(subscription => subscription.name)++ expect(subscriptionIds).to.include(unrelatedSubscription.id)+ expect(subscriptionIds).to.include(unrelatedSubscriptionWithActions.id)+ expect(subscriptionNames).to.include('Another watched words list')+ expect(subscriptionNames).to.include('Another watched words list with actions')+ }++ {+ const { total, data } = await server.watchedWordsLists.listWordsLists({ token, accountName })++ expect(total).to.equal(2)+ expect(data).to.have.lengthOf(2)++ const manualList = data.find(list => list.listName === 'Manual watched words list')+ const unrelatedImportedList = data.find(list => list.listName === 'Another watched words list with actions')++ expect(manualList).to.not.be.undefined+ expect(unrelatedImportedList).to.not.be.undefined+ expect(manualList.words).to.deep.equal([ 'manual-word' ])+ expect(manualList.subscriptionUrl).to.be.null+ expect(unrelatedImportedList.words).to.deep.equal([ 'unrelated-word' ])+ expect(unrelatedImportedList.subscriptionUrl).to.equal(`http://127.0.0.1:${mockPort}/list-3`)+ }+ })+ }++ describe('Managing account watched words subscriptions', function () {+ runTests('account')+ })++ describe('Managing instance watched words subscriptions', function () {+ runTests('server')+ })++ after(async function () {+ await cleanupTests([ server ])+ await mockServer.terminate()+ })+})diff --git a/packages/tests/src/api/moderation/watched-words.ts b/packages/tests/src/api/moderation/watched-words.tsindex c6c138ee5a..bbf47a5b07 100644--- a/packages/tests/src/api/moderation/watched-words.ts+++ b/packages/tests/src/api/moderation/watched-words.ts@@ -87,6 +87,7 @@ describe('Test watched words', function () {expect(data[0].updatedAt).to.existexpect(data[0].listName).to.equal('list one')expect(data[0].words).to.deep.equal([ 'word4', 'word5' ])+ expect(data[0].subscriptionUrl).to.be.null}// With sort, start, count@@ -102,9 +103,11 @@ describe('Test watched words', function () {expect(data[0].listName).to.equal('list user one')expect(data[0].words).to.deep.equal([ 'word1' ])+ expect(data[0].subscriptionUrl).to.be.nullexpect(data[1].listName).to.equal('list user two')expect(data[1].words).to.deep.equal([ 'word2', 'word3' ])+ expect(data[1].subscriptionUrl).to.be.null}{@@ -157,6 +160,7 @@ describe('Test watched words', function () {expect(list.listName).to.equal('updated-list')expect(list.words).to.deep.equal([ 'updated-word1', 'updated-word2' ])+ expect(list.subscriptionUrl).to.be.null})it('Should delete watched words lists', async function () {diff --git a/packages/tests/src/shared/mock-servers/mock-watched-words.ts b/packages/tests/src/shared/mock-servers/mock-watched-words.tsnew file mode 100644index 0000000000..100b11ad17--- /dev/null+++ b/packages/tests/src/shared/mock-servers/mock-watched-words.ts@@ -0,0 +1,78 @@+import express from 'express'+import { Server } from 'http'+import { getPort, randomListen, terminateServer } from './shared.js'++export class MockWatchedWords {+ private server: Server++ private name = 'Remote watched words'+ private actions: { type: 'add' | 'remove', word: string, createdAt: string }[] = []++ async initialize () {+ const app = express()++ app.get([ '/list.json', '/list-copy.json' ], (_req: express.Request, res: express.Response) => {+ return res.json({+ name: this.name,+ actions: this.actions+ })+ })++ app.get([ '/list-2', '/list-2-copy' ], (_req: express.Request, res: express.Response) => {+ return res.json({+ name: 'Another watched words list',+ actions: []+ })+ })++ app.get([ '/list-3', '/list-3-copy' ], (_req: express.Request, res: express.Response) => {+ return res.json({+ name: 'Another watched words list with actions',+ actions: [+ {+ type: 'add',+ word: 'unrelated-word',+ createdAt: new Date('2020-01-01T00:00:00.000Z').toISOString()+ }+ ]+ })+ })++ app.get('/invalid-watched-words-1', (_req: express.Request, res: express.Response) => {+ return res.json({ name: '', actions: [] })+ })++ app.get('/invalid-watched-words-2', (_req: express.Request, res: express.Response) => {+ return res.send('')+ })++ app.get('/too-many-watched-words', (_req: express.Request, res: express.Response) => {+ const actions = Array.from({ length: 501 }, (_, i) => ({+ type: 'add' as const,+ word: `word-${i}`,+ createdAt: new Date(`2020-01-01T00:00:${(i % 60).toString().padStart(2, '0')}.000Z`).toISOString()+ }))++ return res.json({+ name: 'Too many watched words',+ actions+ })+ })++ this.server = await randomListen(app)++ return getPort(this.server)+ }++ setName (name: string) {+ this.name = name+ }++ setActions (actions: { type: 'add' | 'remove', word: string, createdAt: string }[]) {+ this.actions = actions+ }++ terminate () {+ return terminateServer(this.server)+ }+}diff --git a/packages/typescript-utils/src/types.ts b/packages/typescript-utils/src/types.tsindex a15ee03ba4..bf332fc8e5 100644--- a/packages/typescript-utils/src/types.ts+++ b/packages/typescript-utils/src/types.ts@@ -45,10 +45,17 @@ export type Unpacked<T> = T extends (infer U)[] ? U : Texport type Awaitable<T> = T | PromiseLike<T>// Thanks https://stackoverflow.com/a/52761156-export type OverloadedParameters<T> =- // oxlint-disable-next-line max-len- T extends { (...args: infer A1): any, (...args: infer A2): any, (...args: infer A3): any, (...args: infer A4): any } ? A1 | A2 | A3 | A4 :- T extends { (...args: infer A1): any, (...args: infer A2): any, (...args: infer A3): any } ? A1 | A2 | A3 :- T extends { (...args: infer A1): any, (...args: infer A2): any } ? A1 | A2 :- T extends (...args: infer A) => any ? A- : any+export type OverloadedParameters<T> = T extends+ { (...args: infer A1): any, (...args: infer A2): any, (...args: infer A3): any, (...args: infer A4): any } ? A1 | A2 | A3 | A4 :+ T extends { (...args: infer A1): any, (...args: infer A2): any, (...args: infer A3): any } ? A1 | A2 | A3 :+ T extends { (...args: infer A1): any, (...args: infer A2): any } ? A1 | A2 :+ T extends (...args: infer A) => any ? A+ : any++export function arrayOfAllFactory<T> () {+ return function <U extends T[]> (+ array: U & ([T] extends [U[number]] ? unknown : 'Missing members')+ ) {+ return array+ }+}diff --git a/server/core/controllers/api/automatic-tags.ts b/server/core/controllers/api/automatic-tags.tsindex e107e3101c..d48140aa4d 100644--- a/server/core/controllers/api/automatic-tags.ts+++ b/server/core/controllers/api/automatic-tags.ts@@ -1,30 +1,24 @@import { AutomaticTagPolicy, CommentAutomaticTagPoliciesUpdate, HttpStatusCode, UserRight } from '@peertube/peertube-models'import { AutomaticTagger } from '@server/lib/automatic-tags/automatic-tagger.js'import { setAccountAutomaticTagsPolicy } from '@server/lib/automatic-tags/automatic-tags.js'-import {- manageAccountAutomaticTagsValidator,- updateAutomaticTagPoliciesValidator-} from '@server/middlewares/validators/automatic-tags.js'-import { getServerActor } from '@server/models/application/application.js'+import { manageAccountAutomaticTagsValidator, updateAutomaticTagPoliciesValidator } from '@server/middlewares/validators/automatic-tags.js'+import { getServerAccount } from '@server/models/application/application.js'import express from 'express'-import {- apiRateLimiter,- asyncMiddleware,- authenticate,- ensureUserHasRight-} from '../../middlewares/index.js'+import { apiRateLimiter, asyncMiddleware, authenticate, ensureUserHasRight } from '../../middlewares/index.js'const automaticTagRouter = express.Router()automaticTagRouter.use(apiRateLimiter)-automaticTagRouter.get('/policies/accounts/:accountName/comments',+automaticTagRouter.get(+ '/policies/accounts/:accountName/comments',authenticate,asyncMiddleware(manageAccountAutomaticTagsValidator),asyncMiddleware(getAutomaticTagPolicies))-automaticTagRouter.put('/policies/accounts/:accountName/comments',+automaticTagRouter.put(+ '/policies/accounts/:accountName/comments',authenticate,asyncMiddleware(manageAccountAutomaticTagsValidator),asyncMiddleware(updateAutomaticTagPoliciesValidator),@@ -33,13 +27,15 @@ automaticTagRouter.put('/policies/accounts/:accountName/comments',// ----------------------------------------------------------------------------automaticTagRouter.get('/accounts/:accountName/available',+automaticTagRouter.get(+ '/accounts/:accountName/available',authenticate,asyncMiddleware(manageAccountAutomaticTagsValidator),asyncMiddleware(getAccountAutomaticTagAvailable))-automaticTagRouter.get('/server/available',+automaticTagRouter.get(+ '/server/available',authenticate,ensureUserHasRight(UserRight.MANAGE_INSTANCE_AUTO_TAGS),asyncMiddleware(getServerAutomaticTagAvailable)@@ -76,7 +72,7 @@ async function getAccountAutomaticTagAvailable (req: express.Request, res: expre}async function getServerAutomaticTagAvailable (req: express.Request, res: express.Response) {- const result = await AutomaticTagger.getAutomaticTagAvailable((await getServerActor()).Account)+ const result = await AutomaticTagger.getAutomaticTagAvailable(await getServerAccount())return res.json(result)}diff --git a/server/core/controllers/api/videos/update.ts b/server/core/controllers/api/videos/update.tsindex 5fc5652cc1..f9a6b42918 100644--- a/server/core/controllers/api/videos/update.ts+++ b/server/core/controllers/api/videos/update.ts@@ -21,6 +21,7 @@ import { VideoPathManager } from '@server/lib/video-path-manager.js'import { setVideoPrivacy } from '@server/lib/video-privacy.js'import { setVideoTags } from '@server/lib/video.js'import { openapiOperationDoc } from '@server/middlewares/doc.js'+import { getServerAccount } from '@server/models/application/application.js'import { VideoChannelActivityModel } from '@server/models/video/video-channel-activity.js'import { VideoPasswordModel } from '@server/models/video/video-password.js'import { FilteredModelAttributes } from '@server/types/index.js'@@ -194,8 +195,12 @@ async function updateVideo (req: express.Request, res: express.Response) {}if (oldName !== video.name || oldDescription !== video.description) {- const automaticTags = await new AutomaticTagger().buildVideoAutomaticTags({ video, transaction: t })- await setAndSaveVideoAutomaticTags({ video, automaticTags, transaction: t })+ const automaticTagsByAccount = await new AutomaticTagger().buildVideoAutomaticTags({+ serverAccount: await getServerAccount(),+ video,+ transaction: t+ })+ await setAndSaveVideoAutomaticTags({ video, automaticTagsByAccount, transaction: t })}await autoBlacklistVideoIfNeeded({diff --git a/server/core/controllers/api/watched-words.ts b/server/core/controllers/api/watched-words.tsindex 800257dad9..ff78ee58fb 100644--- a/server/core/controllers/api/watched-words.ts+++ b/server/core/controllers/api/watched-words.ts@@ -1,29 +1,46 @@import { HttpStatusCode, UserRight } from '@peertube/peertube-models'import { Awaitable } from '@peertube/peertube-typescript-utils'+import { deleteInTransactionWithRetries } from '@server/helpers/database-utils.js'+import { logger } from '@server/helpers/logger.js'+import { sequelizeTypescript } from '@server/initializers/database.js'+import { createRebuildAutomaticTagsJob } from '@server/lib/automatic-tags/automatic-tags.js'+import { fetchAndValidateWatchedWordsSubscriptionActions } from '@server/lib/watched-words-subscriptions.js'+import {+ addWatchedWordsSubscriptionValidatorFactory,+ deleteWatchedWordsSubscriptionValidatorFactory+} from '@server/middlewares/validators/watched-words-subscriptions.js'import {addWatchedWordsListValidatorFactory,getWatchedWordsListValidatorFactory,manageAccountWatchedWordsListValidator,updateWatchedWordsListValidatorFactory} from '@server/middlewares/validators/watched-words.js'-import { getServerActor } from '@server/models/application/application.js'+import { getServerAccount } from '@server/models/application/application.js'import { WatchedWordsListModel } from '@server/models/watched-words/watched-words-list.js'+import { WatchedWordsSubscriptionModel } from '@server/models/watched-words/watched-words-subscription.js'import { MAccountId } from '@server/types/models/index.js'import express from 'express'import { getFormattedObjects } from '../../helpers/utils.js'import {apiRateLimiter,asyncMiddleware,- authenticate, ensureUserHasRight, paginationValidator,+ authenticate,+ ensureUserHasRight,+ paginationValidator,setDefaultPagination,setDefaultSort,- watchedWordsListsSortValidator+ watchedWordsListsSortValidator,+ watchedWordsSubscriptionsSortValidator} from '../../middlewares/index.js'const watchedWordsRouter = express.Router()watchedWordsRouter.use(apiRateLimiter)+// ---------------------------------------------------------------------------+// Watched words lists list routes+// ---------------------------------------------------------------------------+{const common = [authenticate,@@ -33,43 +50,122 @@ watchedWordsRouter.use(apiRateLimiter)setDefaultPagination]- watchedWordsRouter.get('/accounts/:accountName/lists',+ watchedWordsRouter.get(+ '/accounts/:accountName/lists',...common,-asyncMiddleware(manageAccountWatchedWordsListValidator),asyncMiddleware(listWatchedWordsListsFactory(res => res.locals.account)))- watchedWordsRouter.get('/server/lists',+ watchedWordsRouter.get(+ '/server/lists',+ ...common,+ ensureUserHasRight(UserRight.MANAGE_INSTANCE_WATCHED_WORDS),+ asyncMiddleware(listWatchedWordsListsFactory(() => getServerAccount()))+ )+}++// ---------------------------------------------------------------------------+// Watched words subscriptions list routes+// ---------------------------------------------------------------------------++{+ const common = [+ authenticate,+ paginationValidator,+ watchedWordsSubscriptionsSortValidator,+ setDefaultSort,+ setDefaultPagination+ ]++ watchedWordsRouter.get(+ '/accounts/:accountName/subscriptions',+ ...common,+ asyncMiddleware(manageAccountWatchedWordsListValidator),+ asyncMiddleware(listWatchedWordsSubscriptionsFactory(res => res.locals.account))+ )++ watchedWordsRouter.get(+ '/server/subscriptions',...common,+ ensureUserHasRight(UserRight.MANAGE_INSTANCE_WATCHED_WORDS),+ asyncMiddleware(listWatchedWordsSubscriptionsFactory(() => getServerAccount()))+ )+}++// ---------------------------------------------------------------------------+// Watched words subscriptions create routes+// ---------------------------------------------------------------------------++{+ watchedWordsRouter.post(+ '/accounts/:accountName/subscriptions',+ authenticate,+ asyncMiddleware(manageAccountWatchedWordsListValidator),+ asyncMiddleware(addWatchedWordsSubscriptionValidatorFactory(res => res.locals.account)),+ asyncMiddleware(addWatchedWordsSubscriptionFactory(res => res.locals.account))+ )+ watchedWordsRouter.post(+ '/server/subscriptions',+ authenticate,ensureUserHasRight(UserRight.MANAGE_INSTANCE_WATCHED_WORDS),- asyncMiddleware(listWatchedWordsListsFactory(() => getServerActor().then(a => a.Account)))+ asyncMiddleware(addWatchedWordsSubscriptionValidatorFactory(() => getServerAccount())),+ asyncMiddleware(addWatchedWordsSubscriptionFactory(() => getServerAccount())))}+// ---------------------------------------------------------------------------+// Watched words subscriptions delete routes// ---------------------------------------------------------------------------{- watchedWordsRouter.post('/accounts/:accountName/lists',+ watchedWordsRouter.delete(+ '/accounts/:accountName/subscriptions/:id',+ authenticate,+ asyncMiddleware(manageAccountWatchedWordsListValidator),+ asyncMiddleware(deleteWatchedWordsSubscriptionValidatorFactory(res => res.locals.account)),+ asyncMiddleware(deleteWatchedWordsSubscription)+ )++ watchedWordsRouter.delete(+ '/server/subscriptions/:id',+ authenticate,+ ensureUserHasRight(UserRight.MANAGE_INSTANCE_WATCHED_WORDS),+ asyncMiddleware(deleteWatchedWordsSubscriptionValidatorFactory(() => getServerAccount())),+ asyncMiddleware(deleteWatchedWordsSubscription)+ )+}++// ---------------------------------------------------------------------------+// Watched words lists create routes+// ---------------------------------------------------------------------------++{+ watchedWordsRouter.post(+ '/accounts/:accountName/lists',authenticate,asyncMiddleware(manageAccountWatchedWordsListValidator),asyncMiddleware(addWatchedWordsListValidatorFactory(res => res.locals.account)),asyncMiddleware(addWatchedWordsListFactory(res => res.locals.account)))- watchedWordsRouter.post('/server/lists',+ watchedWordsRouter.post(+ '/server/lists',authenticate,ensureUserHasRight(UserRight.MANAGE_INSTANCE_WATCHED_WORDS),- asyncMiddleware(addWatchedWordsListValidatorFactory(() => getServerActor().then(a => a.Account))),- asyncMiddleware(addWatchedWordsListFactory(() => getServerActor().then(a => a.Account)))+ asyncMiddleware(addWatchedWordsListValidatorFactory(() => getServerAccount())),+ asyncMiddleware(addWatchedWordsListFactory(() => getServerAccount())))}+// ---------------------------------------------------------------------------+// Watched words lists update routes// ---------------------------------------------------------------------------{- watchedWordsRouter.put('/accounts/:accountName/lists/:listId',+ watchedWordsRouter.put(+ '/accounts/:accountName/lists/:listId',authenticate,asyncMiddleware(manageAccountWatchedWordsListValidator),asyncMiddleware(getWatchedWordsListValidatorFactory(res => res.locals.account)),@@ -77,39 +173,48 @@ watchedWordsRouter.use(apiRateLimiter)asyncMiddleware(updateWatchedWordsList))- watchedWordsRouter.put('/server/lists/:listId',+ watchedWordsRouter.put(+ '/server/lists/:listId',authenticate,ensureUserHasRight(UserRight.MANAGE_INSTANCE_WATCHED_WORDS),- asyncMiddleware(getWatchedWordsListValidatorFactory(() => getServerActor().then(a => a.Account))),- asyncMiddleware(updateWatchedWordsListValidatorFactory(() => getServerActor().then(a => a.Account))),+ asyncMiddleware(getWatchedWordsListValidatorFactory(() => getServerAccount())),+ asyncMiddleware(updateWatchedWordsListValidatorFactory(() => getServerAccount())),asyncMiddleware(updateWatchedWordsList))}+// ---------------------------------------------------------------------------+// Watched words lists delete routes// ---------------------------------------------------------------------------{- watchedWordsRouter.delete('/accounts/:accountName/lists/:listId',+ watchedWordsRouter.delete(+ '/accounts/:accountName/lists/:listId',authenticate,asyncMiddleware(manageAccountWatchedWordsListValidator),asyncMiddleware(getWatchedWordsListValidatorFactory(res => res.locals.account)),asyncMiddleware(deleteWatchedWordsList))- watchedWordsRouter.delete('/server/lists/:listId',+ watchedWordsRouter.delete(+ '/server/lists/:listId',authenticate,ensureUserHasRight(UserRight.MANAGE_INSTANCE_WATCHED_WORDS),- asyncMiddleware(getWatchedWordsListValidatorFactory(() => getServerActor().then(a => a.Account))),+ asyncMiddleware(getWatchedWordsListValidatorFactory(() => getServerAccount())),asyncMiddleware(deleteWatchedWordsList))}+// ---------------------------------------------------------------------------+// Router export// ---------------------------------------------------------------------------export {watchedWordsRouter}+// ---------------------------------------------------------------------------+// Factories and handlers// ---------------------------------------------------------------------------function listWatchedWordsListsFactory (accountGetter: (res: express.Response) => Awaitable<MAccountId>) {@@ -127,13 +232,17 @@ function listWatchedWordsListsFactory (accountGetter: (res: express.Response) =>function addWatchedWordsListFactory (accountGetter: (res: express.Response) => Awaitable<MAccountId>) {return async (req: express.Request, res: express.Response) => {+ const account = await accountGetter(res)+const list = await WatchedWordsListModel.createList({- accountId: (await accountGetter(res)).id,+ accountId: account.id,listName: req.body.listName,words: req.body.words})+ await createRebuildAutomaticTagsJob({ accountId: account.id })+return res.json({watchedWordsList: {id: list.id@@ -142,6 +251,55 @@ function addWatchedWordsListFactory (accountGetter: (res: express.Response) => A}}+function listWatchedWordsSubscriptionsFactory (accountGetter: (res: express.Response) => Awaitable<MAccountId>) {+ return async (req: express.Request, res: express.Response) => {+ const resultList = await WatchedWordsSubscriptionModel.listForApi({+ accountId: (await accountGetter(res)).id,+ start: req.query.start,+ count: req.query.count,+ sort: req.query.sort,+ search: req.query.search+ })++ return res.json(getFormattedObjects(resultList.data, resultList.total))+ }+}++function addWatchedWordsSubscriptionFactory (accountGetter: (res: express.Response) => Awaitable<MAccountId>) {+ return async (req: express.Request, res: express.Response) => {+ const account = await accountGetter(res)++ try {+ const { name } = await fetchAndValidateWatchedWordsSubscriptionActions(req.body.url)++ const existing = await WatchedWordsListModel.loadByListName({ accountId: account.id, listName: name })+ if (existing) {+ return res.fail({+ status: HttpStatusCode.CONFLICT_409,+ message: req.t('A watched words list with name {name} already exists', { name })+ })+ }++ const subscription = await WatchedWordsSubscriptionModel.create({+ accountId: account.id,+ name,+ url: req.body.url,+ lastSyncAt: null,+ lastActionCreatedAt: null+ })++ return res.json(subscription.toFormattedJSON())+ } catch (err) {+ logger.warn('Failed to fetch or parse watched words URL when adding watched word subscription.', { url: req.body.url, err })++ res.fail({+ status: HttpStatusCode.BAD_REQUEST_400,+ message: req.t('Cannot fetch or parse watched words URL')+ })+ }+ }+}+async function updateWatchedWordsList (req: express.Request, res: express.Response) {const list = res.locals.watchedWordsList@@ -150,13 +308,32 @@ async function updateWatchedWordsList (req: express.Request, res: express.Responwords: req.body.words})+ await createRebuildAutomaticTagsJob({ accountId: list.accountId })+return res.sendStatus(HttpStatusCode.NO_CONTENT_204)}async function deleteWatchedWordsList (req: express.Request, res: express.Response) {const list = res.locals.watchedWordsList- await list.destroy()+ await deleteInTransactionWithRetries(list)++ await createRebuildAutomaticTagsJob({ accountId: list.accountId })++ return res.sendStatus(HttpStatusCode.NO_CONTENT_204)+}++async function deleteWatchedWordsSubscription (req: express.Request, res: express.Response) {+ const subscription = res.locals.watchedWordsSubscription++ await sequelizeTypescript.transaction(async transaction => {+ await WatchedWordsListModel.removeImportedBySubscription({+ accountId: subscription.accountId,+ watchedWordsSubscriptionId: subscription.id,+ transaction+ })+ await subscription.destroy({ transaction })+ })return res.sendStatus(HttpStatusCode.NO_CONTENT_204)}diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.tsindex 02403107bf..c8aff53310 100644--- a/server/core/initializers/constants.ts+++ b/server/core/initializers/constants.ts@@ -62,7 +62,7 @@ import { CONFIG, registerConfigChangedHandler } from './config.js'// ----------------------------------------------------------------------------export const LAST_MIGRATION_VERSION = 1040+export const LAST_MIGRATION_VERSION = 1045// ---------------------------------------------------------------------------@@ -158,6 +158,7 @@ export const SORTABLE_COLUMNS = {BLOCKLIST_SUBSCRIPTIONS: [ 'name', 'createdAt', 'lastSyncAt' ],WATCHED_WORDS_LISTS: [ 'createdAt', 'updatedAt', 'listName' ],+ WATCHED_WORDS_SUBSCRIPTIONS: [ 'name', 'createdAt', 'lastSyncAt' ],USER_NOTIFICATIONS: [ 'createdAt', 'read' ],@@ -212,6 +213,7 @@ export const REMOTE_SCHEME = {// ---------------------------------------------------------------------------export const JOB_ATTEMPTS: { [id in JobType]: number } = {+ 'build-automatic-tags': 1,'activitypub-http-broadcast': 1,'activitypub-http-broadcast-parallel': 1,'activitypub-http-unicast': 1,@@ -243,6 +245,7 @@ export const JOB_ATTEMPTS: { [id in JobType]: number } = {}// Excluded keys are jobs that can be configured by adminsexport const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-import'>]: number } = {+ 'build-automatic-tags': 1,'activitypub-http-broadcast': 1,'activitypub-http-broadcast-parallel': 30,'activitypub-http-unicast': 30,@@ -271,6 +274,7 @@ export const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'v'video-transcription': 1}export const JOB_TTL: { [id in JobType]: number } = {+ 'build-automatic-tags': 1000 * 60 * 30, // 30 minutes'activitypub-http-broadcast': 60000 * 10, // 10 minutes'activitypub-http-broadcast-parallel': 60000 * 10, // 10 minutes'activitypub-http-unicast': 60000 * 10, // 10 minutes@@ -376,7 +380,8 @@ export const SCHEDULER_INTERVALS_MS = {UPDATE_INBOX_STATS: 1000 * 60, // 1 minuteREMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60, // 1 hourCHANNEL_SYNC_CHECK_INTERVAL: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.CHECK_INTERVAL,- BLOCKLIST_SUBSCRIPTIONS_SYNC: 60000 * 60 // 1 hour+ BLOCKLIST_SUBSCRIPTIONS_SYNC: 60000 * 60, // 1 hour+ WATCHED_WORDS_SUBSCRIPTIONS_SYNC: 60000 * 60 // 1 hour}// ---------------------------------------------------------------------------@@ -1293,6 +1298,7 @@ if (process.env.PRODUCTION_CONSTANTS !== 'true') {SCHEDULER_INTERVALS_MS.UPDATE_VIDEOS = 5000SCHEDULER_INTERVALS_MS.AUTO_FOLLOW_INDEX_INSTANCES = 5000SCHEDULER_INTERVALS_MS.BLOCKLIST_SUBSCRIPTIONS_SYNC = 5000+ SCHEDULER_INTERVALS_MS.WATCHED_WORDS_SUBSCRIPTIONS_SYNC = 5000SCHEDULER_INTERVALS_MS.UPDATE_INBOX_STATS = 5000SCHEDULER_INTERVALS_MS.CHECK_PEERTUBE_VERSION = 2000SCHEDULER_INTERVALS_MS.UPDATE_TOKEN_SESSION = 2000diff --git a/server/core/initializers/database.ts b/server/core/initializers/database.tsindex 09a320ee92..bc8febbeab 100644--- a/server/core/initializers/database.ts+++ b/server/core/initializers/database.ts@@ -33,6 +33,7 @@ import { VideoLiveSessionModel } from '@server/models/video/video-live-session.jimport { VideoPasswordModel } from '@server/models/video/video-password.js'import { VideoSourceModel } from '@server/models/video/video-source.js'import { WatchedWordsListModel } from '@server/models/watched-words/watched-words-list.js'+import { WatchedWordsSubscriptionModel } from '@server/models/watched-words/watched-words-subscription.js'import { readFileSync } from 'fs'import pg from 'pg'import { QueryTypes, Transaction } from 'sequelize'@@ -211,6 +212,7 @@ export async function initDatabaseModels (silent: boolean) {CommentAutomaticTagModel,AutomaticTagModel,WatchedWordsListModel,+ WatchedWordsSubscriptionModel,AccountAutomaticTagPolicyModel,UploadImageModel,VideoLiveScheduleModel,diff --git a/server/core/initializers/migrations/1045-watched-words-subscriptions.ts b/server/core/initializers/migrations/1045-watched-words-subscriptions.tsnew file mode 100644index 0000000000..027d89c39c--- /dev/null+++ b/server/core/initializers/migrations/1045-watched-words-subscriptions.ts@@ -0,0 +1,44 @@+import * as Sequelize from 'sequelize'++async function up (utils: {+ transaction: Sequelize.Transaction+ queryInterface: Sequelize.QueryInterface+ sequelize: Sequelize.Sequelize+}): Promise<void> {+ const { transaction } = utils++ {+ const query = `+CREATE TABLE IF NOT EXISTS "watchedWordsSubscription" (+ "id" SERIAL,+ "name" VARCHAR(255) NOT NULL,+ "url" VARCHAR(255) NOT NULL,+ "lastSyncAt" TIMESTAMP WITH TIME ZONE,+ "lastActionCreatedAt" TIMESTAMP WITH TIME ZONE,+ "state" INTEGER NOT NULL DEFAULT 1,+ "accountId" INTEGER NOT NULL REFERENCES "account" ("id") ON DELETE CASCADE ON UPDATE CASCADE,+ "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL,+ "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL,+ PRIMARY KEY ("id")+);`++ await utils.sequelize.query(query, { transaction })+ }++ {+ await utils.sequelize.query(+ `ALTER TABLE "watchedWordsList" ` ++ `ADD COLUMN IF NOT EXISTS "watchedWordsSubscriptionId" INTEGER REFERENCES "watchedWordsSubscription" ("id") ON DELETE CASCADE ON UPDATE CASCADE`,+ { transaction }+ )+ }+}++function down () {+ throw new Error('Not implemented.')+}++export {+ up,+ down+}diff --git a/server/core/lib/activitypub/video-comments.ts b/server/core/lib/activitypub/video-comments.tsindex f23fc860ef..1080dff7ae 100644--- a/server/core/lib/activitypub/video-comments.ts+++ b/server/core/lib/activitypub/video-comments.ts@@ -1,4 +1,6 @@import { HttpStatusCode, VideoCommentPolicy } from '@peertube/peertube-models'+import { CONFIG } from '@server/initializers/config.js'+import { getServerAccount } from '@server/models/application/application.js'import Bluebird from 'bluebird'import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments.js'import { logger } from '../../helpers/logger.js'@@ -20,7 +22,6 @@ import { fetchAP } from './activity.js'import { getOrCreateAPActor } from './actors/index.js'import { checkUrlsSameHost } from './url.js'import { canVideoBeFederated, getOrCreateAPVideo } from './videos/index.js'-import { CONFIG } from '@server/initializers/config.js'type ResolveThreadParams = {url: string@@ -130,10 +131,10 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {return undefined}- const firstReplyAutomaticTags = await getAutomaticTagsAndAssignReview(firstReply, video)+ const firstReplyAutomaticTagsByAccount = await getAutomaticTagsAndAssignReview(firstReply, video)comments[comments.length - 1] = await firstReply.save()- await setAndSaveCommentAutomaticTags({ comment: firstReply, automaticTags: firstReplyAutomaticTags })+ await setAndSaveCommentAutomaticTags({ comment: firstReply, automaticTagsByAccount: firstReplyAutomaticTagsByAccount })for (let i = comments.length - 2; i >= 0; i--) {const comment = comments[i] as MCommentOwnerVideo@@ -147,11 +148,11 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {return undefined}- const automaticTags = await getAutomaticTagsAndAssignReview(comment, video)+ const automaticTagsByAccount = await getAutomaticTagsAndAssignReview(comment, video)comments[i] = await comment.save()- await setAndSaveCommentAutomaticTags({ comment, automaticTags })+ await setAndSaveCommentAutomaticTags({ comment, automaticTagsByAccount })}resultComment = comments[0] as MCommentOwnerVideo@@ -160,22 +161,31 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {return { video, comment: resultComment, commentCreated }}+// Return the automatic tags by account id (owner and server accounts) and assign the held for review status to the comment if neededasync function getAutomaticTagsAndAssignReview (comment: MComment, video: MVideoAccountLight) {// Remote comment already exists in database or remote video -> we don't need to rebuild automatic tagsif (comment.id) return []const ownerAccount = video.VideoChannel.Account- const automaticTags = await new AutomaticTagger().buildCommentsAutomaticTags({ ownerAccount, text: comment.text })+ const automaticTagsByAccount = await new AutomaticTagger().buildCommentsAutomaticTags({+ serverAccount: await getServerAccount(),+ ownerAccount,+ text: comment.text+ })// Third parties rely on origin, so if origin has the comment it's not held for reviewif (video.isLocal() || comment.isLocal()) {- comment.heldForReview = await shouldCommentBeHeldForReview({ user: null, video, automaticTags })+ comment.heldForReview = await shouldCommentBeHeldForReview({+ user: null,+ video,+ ownerAutomaticTags: automaticTagsByAccount[video.VideoChannel.accountId]+ })} else {comment.heldForReview = false}- return automaticTags+ return automaticTagsByAccount}// ---------------------------------------------------------------------------diff --git a/server/core/lib/activitypub/videos/shared/abstract-builder.ts b/server/core/lib/activitypub/videos/shared/abstract-builder.tsindex 4946f1b32f..4f234d4a3d 100644--- a/server/core/lib/activitypub/videos/shared/abstract-builder.ts+++ b/server/core/lib/activitypub/videos/shared/abstract-builder.ts@@ -9,6 +9,7 @@ import { setAndSaveVideoAutomaticTags } from '@server/lib/automatic-tags/automatimport { updateRemoteVideoThumbnail } from '@server/lib/thumbnail.js'import { replaceChapters } from '@server/lib/video-chapters.js'import { setVideoTags } from '@server/lib/video.js'+import { getServerAccount } from '@server/models/application/application.js'import { StoryboardModel } from '@server/models/video/storyboard.js'import { VideoCaptionModel } from '@server/models/video/video-caption.js'import { VideoFileModel } from '@server/models/video/video-file.js'@@ -243,7 +244,11 @@ export abstract class APVideoAbstractBuilder {if (video.name === oldVideo?.name && video.description === oldVideo.description) return- const automaticTags = await new AutomaticTagger().buildVideoAutomaticTags({ video, transaction })- await setAndSaveVideoAutomaticTags({ video, automaticTags, transaction })+ const automaticTagsByAccount = await new AutomaticTagger().buildVideoAutomaticTags({+ serverAccount: await getServerAccount(),+ video,+ transaction+ })+ await setAndSaveVideoAutomaticTags({ video, automaticTagsByAccount, transaction })}}diff --git a/server/core/lib/automatic-tags/automatic-tagger.ts b/server/core/lib/automatic-tags/automatic-tagger.tsindex 8f0475533f..aaee5a15ed 100644--- a/server/core/lib/automatic-tags/automatic-tagger.ts+++ b/server/core/lib/automatic-tags/automatic-tagger.ts@@ -1,7 +1,6 @@import { AutomaticTagAvailable, AutomaticTagPolicy, CommentAutomaticTagPolicies } from '@peertube/peertube-models'import { logger, loggerTagsFactory } from '@server/helpers/logger.js'import { WEBSERVER } from '@server/initializers/constants.js'-import { getServerActor } from '@server/models/application/application.js'import { AccountAutomaticTagPolicyModel } from '@server/models/automatic-tag/account-automatic-tag-policy.js'import { WatchedWordsListModel } from '@server/models/watched-words/watched-words-list.js'import { MAccount, MAccountId, MVideo } from '@server/types/models/index.js'@@ -13,43 +12,46 @@ const lTags = loggerTagsFactory('automatic-tags')const linkifyit = new Linkifyit()export class AutomaticTagger {-private static readonly SPECIAL_TAGS = {EXTERNAL_LINK: 'external-link'}async buildCommentsAutomaticTags (options: {- ownerAccount: MAccount+ serverAccount: MAccount | null+ ownerAccount: MAccount | nulltext: stringtransaction?: Transaction}) {- const { text, ownerAccount, transaction } = options+ const { text, serverAccount, ownerAccount, transaction } = options- const serverAccount = (await getServerActor()).Account+ // accountId -> tags+ const result: Record<number, string[]> = {}try {- const [ accountTags, serverTags ] = await Promise.all([- this.buildAutomaticTags({ account: ownerAccount, text, transaction }),- this.buildAutomaticTags({ account: serverAccount, text, transaction })- ])+ if (serverAccount) {+ result[serverAccount.id] = await this.buildAutomaticTags({ account: serverAccount, text, transaction })+ }- logger.debug('Built automatic tags for comment', { text, accountTags, serverTags, ...lTags() })+ if (ownerAccount) {+ result[ownerAccount.id] = await this.buildAutomaticTags({ account: ownerAccount, text, transaction })+ }- return [ ...accountTags, ...serverTags ]+ logger.debug('Built automatic tags for comment', { text, result, ...lTags() })++ return result} catch (err) {logger.error('Cannot build comment automatic tags', { text, err, ...lTags() })- return []+ return {}}}async buildVideoAutomaticTags (options: {+ serverAccount: MAccountvideo: MVideotransaction?: Transaction}) {- const { video, transaction } = options-- const serverAccount = (await getServerActor()).Account+ const { video, serverAccount, transaction } = optionstry {const [ videoNameTags, videoDescriptionTags ] = await Promise.all([@@ -57,9 +59,15 @@ export class AutomaticTagger {this.buildAutomaticTags({ account: serverAccount, text: video.description, transaction })])- logger.debug('Built automatic tags for video', { video, videoNameTags, videoDescriptionTags, ...lTags() })+ logger.debug('Built automatic tags for video', {+ videoName: video.name,+ videoDescription: video.description,+ videoNameTags,+ videoDescriptionTags,+ ...lTags()+ })- return [ ...videoNameTags, ...videoDescriptionTags ]+ return { [serverAccount.id]: [ ...videoNameTags, ...videoDescriptionTags ] }} catch (err) {logger.error('Cannot build video automatic tags', { video, err, ...lTags() })@@ -75,28 +83,31 @@ export class AutomaticTagger {const { text, account, transaction } = optionsconst tagsDone = new Set<string>()- const automaticTags: { name: string, accountId: number }[] = []+ const automaticTags: string[] = []// Watched words by account that published the videoconst watchedWords = await WatchedWordsListModel.buildWatchedWordsRegexp({ accountId: account.id, transaction })- logger.debug(`Got watched words regex for account ${account.getDisplayName()}`, { watchedWords, ...lTags() })+ logger.debug(`Got watched words regex for account ${account.getDisplayName()}`, {+ listNames: watchedWords.map(r => r.listName),+ ...lTags()+ })for (const { listName, regex } of watchedWords) {try {if (regex.test(text)) {tagsDone.add(listName)- automaticTags.push({ name: listName, accountId: account.id })+ automaticTags.push(listName)}} catch (err) {- logger.error('Cannot test regex against text', { regex, err, ...lTags() })+ logger.error('Cannot test regex against text', { listName, regex: regex.toString(), err, ...lTags() })}}// Core PeerTube tagsif (!tagsDone.has(AutomaticTagger.SPECIAL_TAGS.EXTERNAL_LINK) && this.hasExternalLinks(text)) {// This is a global tag, not assigned to a specific account- automaticTags.push({ name: AutomaticTagger.SPECIAL_TAGS.EXTERNAL_LINK, accountId: account.id })+ automaticTags.push(AutomaticTagger.SPECIAL_TAGS.EXTERNAL_LINK)tagsDone.add(AutomaticTagger.SPECIAL_TAGS.EXTERNAL_LINK)}diff --git a/server/core/lib/automatic-tags/automatic-tags.ts b/server/core/lib/automatic-tags/automatic-tags.tsindex 3ecf79863d..97e67fecdd 100644--- a/server/core/lib/automatic-tags/automatic-tags.ts+++ b/server/core/lib/automatic-tags/automatic-tags.ts@@ -1,4 +1,5 @@import { AutomaticTagPolicyType } from '@peertube/peertube-models'+import { getServerAccount } from '@server/models/application/application.js'import { AccountAutomaticTagPolicyModel } from '@server/models/automatic-tag/account-automatic-tag-policy.js'import { AutomaticTagModel } from '@server/models/automatic-tag/automatic-tag.js'import { CommentAutomaticTagModel } from '@server/models/automatic-tag/comment-automatic-tag.js'@@ -12,24 +13,29 @@ import {MVideoAutomaticTagWithTag} from '@server/types/models/index.js'import { Transaction } from 'sequelize'+import { JobQueue } from '../job-queue/job-queue.js'export async function setAndSaveCommentAutomaticTags (options: {comment: MComment- automaticTags: { accountId: number, name: string }[]+ automaticTagsByAccount: Record<number, string[]>transaction?: Transaction}) {- const { comment, automaticTags, transaction } = options+ const { comment, automaticTagsByAccount, transaction } = options- if (automaticTags.length === 0) return+ const { toCreateItems, toDeleteItems } = await _buildAutomaticTagItems({+ automaticTagsByAccount,+ existingAutomaticTagsGetter: (accountIds: number[]) => {+ return CommentAutomaticTagModel.listByAccountIdsAndCommentId({ commentId: comment.id, accountIds, transaction })+ }+ })- const commentAutomaticTags: MCommentAutomaticTagWithTag[] = []-- const accountIds = new Set(automaticTags.map(t => t.accountId))- for (const accountId of accountIds) {- await CommentAutomaticTagModel.deleteAllOfAccountAndComment({ accountId, commentId: comment.id, transaction })+ for (const item of toDeleteItems) {+ await item.destroy({ transaction })}- for (const tag of automaticTags) {+ const commentAutomaticTags: MCommentAutomaticTagWithTag[] = []++ for (const tag of toCreateItems) {const automaticTagInstance = await AutomaticTagModel.findOrCreateAutomaticTag({ tag: tag.name, transaction })const [ commentAutomaticTag ] = await CommentAutomaticTagModel.upsert({@@ -43,26 +49,31 @@ export async function setAndSaveCommentAutomaticTags (options: {commentAutomaticTags.push(commentAutomaticTag)}- (comment as MCommentAdminOrUserFormattable).CommentAutomaticTags = commentAutomaticTags+ ;(comment as MCommentAdminOrUserFormattable).CommentAutomaticTags = commentAutomaticTags}export async function setAndSaveVideoAutomaticTags (options: {video: MVideo- automaticTags: { accountId: number, name: string }[]+ automaticTagsByAccount: Record<number, string[]>transaction?: Transaction}) {- const { video, automaticTags, transaction } = options+ const { video, automaticTagsByAccount, transaction } = options++ const { toCreateItems, toDeleteItems } = await _buildAutomaticTagItems({+ automaticTagsByAccount,- if (automaticTags.length === 0) return+ existingAutomaticTagsGetter: accountIds => {+ return VideoAutomaticTagModel.listByAccountIdsAndVideoId({ videoId: video.id, accountIds, transaction })+ }+ })- const accountIds = new Set(automaticTags.map(t => t.accountId))- for (const accountId of accountIds) {- await VideoAutomaticTagModel.deleteAllOfAccountAndVideo({ accountId, videoId: video.id, transaction })+ for (const item of toDeleteItems) {+ await item.destroy({ transaction })}const videoAutomaticTags: MVideoAutomaticTagWithTag[] = []- for (const tag of automaticTags) {+ for (const tag of toCreateItems) {const automaticTagInstance = await AutomaticTagModel.findOrCreateAutomaticTag({ tag: tag.name, transaction })const [ videoAutomaticTag ] = await VideoAutomaticTagModel.upsert({@@ -77,6 +88,37 @@ export async function setAndSaveVideoAutomaticTags (options: {}}+async function _buildAutomaticTagItems<T extends MCommentAutomaticTagWithTag | MVideoAutomaticTagWithTag> (options: {+ automaticTagsByAccount: Record<number, string[]>+ existingAutomaticTagsGetter: (accountIds: number[]) => Promise<T[]>+}) {+ const { automaticTagsByAccount, existingAutomaticTagsGetter } = options++ const accountIds = Object.keys(automaticTagsByAccount).map(id => Number(id))++ // Convert automaticTagsByAccount to a flat list of { accountId, name }+ const automaticTags: { accountId: number, name: string }[] = []+ for (const [ accountId, tags ] of Object.entries(automaticTagsByAccount)) {+ for (const tag of tags) {+ automaticTags.push({ accountId: Number(accountId), name: tag })+ }+ }++ const existingVideoAutomaticTags = await existingAutomaticTagsGetter(accountIds)++ const existingByKey = new Map(existingVideoAutomaticTags.map(tag => [ `${tag.accountId}:${tag.AutomaticTag.name}`, tag ]))+ const desiredKeys = new Set(automaticTags.map(tag => `${tag.accountId}:${tag.name}`))++ const toDeleteItems = existingVideoAutomaticTags+ .filter(tag => !desiredKeys.has(`${tag.accountId}:${tag.AutomaticTag.name}`))++ const toCreateItems = automaticTags.filter(tag => !existingByKey.has(`${tag.accountId}:${tag.name}`))++ return { toDeleteItems, toCreateItems }+}++// ---------------------------------------------------------------------------+export async function setAccountAutomaticTagsPolicy (options: {account: MAccountIdtags: string[]@@ -97,3 +139,19 @@ export async function setAccountAutomaticTagsPolicy (options: {}, { transaction })}}++export async function createRebuildAutomaticTagsJob (options: {+ accountId: number+}) {+ const { accountId } = options++ return JobQueue.Instance.createJob({+ type: 'build-automatic-tags',+ payload: {+ accountId,+ ofComments: true,+ ofVideos: (await getServerAccount()).id === accountId+ },+ deduplicationId: `build-automatic-tags:${accountId}`+ })+}diff --git a/server/core/lib/blocklist-subscriptions.ts b/server/core/lib/blocklist-subscriptions.tsindex 4309e8019f..9bf0bff793 100644--- a/server/core/lib/blocklist-subscriptions.ts+++ b/server/core/lib/blocklist-subscriptions.ts@@ -23,6 +23,11 @@ export async function fetchAndValidateAutoMuteList (url: string, startDate?: Datthrow new Error('Invalid auto mute list payload: actions must be an array')}+ const maxActions = 10_000+ if (rawActions.length > maxActions) {+ throw new Error(`Invalid auto mute list payload: actions length must not exceed ${maxActions}`)+ }+const actions: AutoMuteAction[] = rawActions.map((action, index) => validateAutoMuteAction(action, index))return {diff --git a/server/core/lib/blocklist.ts b/server/core/lib/blocklist.tsindex 4d597a3ea5..7ec177607c 100644--- a/server/core/lib/blocklist.ts+++ b/server/core/lib/blocklist.ts@@ -1,7 +1,7 @@import { logger } from '@server/helpers/logger.js'import { sequelizeTypescript } from '@server/initializers/database.js'import { AccountModel } from '@server/models/account/account.js'-import { getServerActor } from '@server/models/application/application.js'+import { getServerAccount } from '@server/models/application/application.js'import { ServerModel } from '@server/models/server/server.js'import { UserNotificationModel } from '@server/models/user/user-notification.js'import {@@ -121,7 +121,7 @@ export function removeServerFromBlocklist (serverBlock: MServerBlocklist) {}export async function isBlockedByServerOrAccount (targetAccount: MAccountHost, userAccount?: MAccountId) {- const serverAccountId = (await getServerActor()).Account.id+ const serverAccountId = (await getServerAccount()).idconst sourceAccounts = [ serverAccountId ]if (userAccount) sourceAccounts.push(userAccount.id)diff --git a/server/core/lib/job-queue/handlers/build-automatic-tags.ts b/server/core/lib/job-queue/handlers/build-automatic-tags.tsnew file mode 100644index 0000000000..630aa87e32--- /dev/null+++ b/server/core/lib/job-queue/handlers/build-automatic-tags.ts@@ -0,0 +1,114 @@+import { BuildAutomaticTagsPayload } from '@peertube/peertube-models'+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'+import { AutomaticTagger } from '@server/lib/automatic-tags/automatic-tagger.js'+import { setAndSaveCommentAutomaticTags, setAndSaveVideoAutomaticTags } from '@server/lib/automatic-tags/automatic-tags.js'+import { getServerAccount } from '@server/models/application/application.js'+import { VideoCommentModel } from '@server/models/video/video-comment.js'+import { VideoModel } from '@server/models/video/video.js'+import { MAccount } from '@server/types/models/index.js'+import { Job } from 'bullmq'++const lTags = loggerTagsFactory('job-queue')++const BATCH_SIZE = 250++export async function processBuildAutomaticTags (job: Job): Promise<void> {+ const payload = job.data as BuildAutomaticTagsPayload++ logger.info('Processing build automatic tags in job %s.', job.id, { payload, ...lTags() })++ const serverAccount = await getServerAccount()+ const accountId = payload.accountId++ let totalRebuiltComments = 0+ let totalRebuiltVideos = 0++ const isServerAccount = accountId === serverAccount.id++ if (payload.ofComments) {+ const rebuiltComments = await rebuildComments({+ videoOwnerId: isServerAccount+ ? null+ : accountId,+ isServerAccount+ })+ totalRebuiltComments += rebuiltComments+ }++ if (payload.ofVideos) {+ if (!isServerAccount) throw new Error('Cannot process video automatic tags for a non-server account.')++ const rebuiltVideos = await rebuildVideos()+ totalRebuiltVideos += rebuiltVideos+ }++ logger.info('Processed build automatic tags in job %s.', job.id, {+ payload,+ rebuiltComments: totalRebuiltComments,+ rebuiltVideos: totalRebuiltVideos,+ ...lTags()+ })+}++async function rebuildComments (options: {+ videoOwnerId: number+ isServerAccount: boolean+}) {+ const { videoOwnerId, isServerAccount } = options++ const automaticTagger = new AutomaticTagger()++ let rebuilt = 0+ let lastId = 0++ while (true) {+ const ids = await VideoCommentModel.batchListIds({ videoOwnerId, deleted: false, batchSize: BATCH_SIZE, lastId })+ if (ids.length === 0) break++ lastId = ids[ids.length - 1]++ for (const id of ids) {+ const comment = await VideoCommentModel.loadByIdWithVideo(id)++ let serverAccount: MAccount+ let ownerAccount: MAccount++ if (isServerAccount) serverAccount = await getServerAccount()+ else ownerAccount = comment.Video.VideoChannel.Account++ const automaticTagsByAccount = await automaticTagger.buildCommentsAutomaticTags({ text: comment.text, serverAccount, ownerAccount })++ await setAndSaveCommentAutomaticTags({ comment, automaticTagsByAccount })++ rebuilt+++ }+ }++ return rebuilt+}++async function rebuildVideos () {+ const automaticTagger = new AutomaticTagger()++ let rebuilt = 0+ let lastId = 0++ while (true) {+ const ids = await VideoModel.batchListIds({ lastId, batchSize: BATCH_SIZE })+ if (ids.length === 0) break++ lastId = ids[ids.length - 1]++ for (const id of ids) {+ const video = await VideoModel.load(id)++ const automaticTagsByAccount = await automaticTagger.buildVideoAutomaticTags({ video, serverAccount: await getServerAccount() })++ await setAndSaveVideoAutomaticTags({ video, automaticTagsByAccount })++ rebuilt+++ }+ }++ return rebuilt+}diff --git a/server/core/lib/job-queue/handlers/video-import.ts b/server/core/lib/job-queue/handlers/video-import.tsindex 8d79666577..1bbf3bf06e 100644--- a/server/core/lib/job-queue/handlers/video-import.ts+++ b/server/core/lib/job-queue/handlers/video-import.ts@@ -27,6 +27,7 @@ import { addLocalOrRemoteStoryboardJobIfNeeded, buildMoveVideoJob } from '@serveimport { VideoPathManager } from '@server/lib/video-path-manager.js'import { buildNextVideoState } from '@server/lib/video-state.js'import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '@server/lib/webtorrent.js'+import { getServerAccount } from '@server/models/application/application.js'import { UserModel } from '@server/models/user/user.js'import { VideoCaptionModel } from '@server/models/video/video-caption.js'import { MUserId, MVideoFile, MVideoFull } from '@server/types/models/index.js'@@ -240,8 +241,12 @@ async function processFile (options: {await replaceChaptersIfNotExist({ video, chapters: containerChapters, transaction: t })- const automaticTags = await new AutomaticTagger().buildVideoAutomaticTags({ video, transaction: t })- await setAndSaveVideoAutomaticTags({ video, automaticTags, transaction: t })+ const automaticTagsByAccount = await new AutomaticTagger().buildVideoAutomaticTags({+ serverAccount: await getServerAccount(),+ video,+ transaction: t+ })+ await setAndSaveVideoAutomaticTags({ video, automaticTagsByAccount, transaction: t })// Now we can federate the video (reload from database, we need more attributes)const videoForFederation = await VideoModel.loadFull(video.uuid, t)diff --git a/server/core/lib/job-queue/job-queue.ts b/server/core/lib/job-queue/job-queue.tsindex 2c49b77ca4..7aeb4f73f2 100644--- a/server/core/lib/job-queue/job-queue.ts+++ b/server/core/lib/job-queue/job-queue.ts@@ -6,6 +6,7 @@ import {ActivitypubHttpUnicastPayload,ActorKeysPayload,AfterVideoChannelImportPayload,+ BuildAutomaticTagsPayload,CreateUserExportPayload,EmailPayload,FederateVideoPayload,@@ -57,6 +58,7 @@ import { processActivityPubHttpUnicast } from './handlers/activitypub-http-unicaimport { refreshAPObject } from './handlers/activitypub-refresher.js'import { processActorKeys } from './handlers/actor-keys.js'import { processAfterVideoChannelImport } from './handlers/after-video-channel-import.js'+import { processBuildAutomaticTags } from './handlers/build-automatic-tags.js'import { processCreateUserExport } from './handlers/create-user-export.js'import { processEmail } from './handlers/email.js'import { processFederateVideo } from './handlers/federate-video.js'@@ -77,6 +79,7 @@ import { processVideoTranscription } from './handlers/video-transcription.js'import { processVideosStats } from './handlers/video-stats.js'export type CreateJobTypeAndPayload =+ | { type: 'build-automatic-tags', payload: BuildAutomaticTagsPayload }| { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload }| { type: 'activitypub-http-broadcast-parallel', payload: ActivitypubHttpBroadcastPayload }| { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload }@@ -114,6 +117,7 @@ export type CreateJobOptions = {}const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {+ 'build-automatic-tags': processBuildAutomaticTags,'activitypub-cleaner': processActivityPubCleaner,'activitypub-follow': processActivityPubFollow,'activitypub-http-broadcast-parallel': processActivityPubParallelHttpBroadcast,@@ -150,6 +154,7 @@ const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> }}const jobTypes: JobType[] = [+ 'build-automatic-tags','activitypub-cleaner','activitypub-follow','activitypub-http-broadcast-parallel',diff --git a/server/core/lib/local-video-creator.ts b/server/core/lib/local-video-creator.tsindex f21e9d0cbf..d198dfc5c8 100644--- a/server/core/lib/local-video-creator.ts+++ b/server/core/lib/local-video-creator.ts@@ -16,6 +16,7 @@ import { retryTransactionWrapper } from '@server/helpers/database-utils.js'import { LoggerTagsFn, logger } from '@server/helpers/logger.js'import { CONFIG } from '@server/initializers/config.js'import { sequelizeTypescript } from '@server/initializers/database.js'+import { getServerAccount } from '@server/models/application/application.js'import { ScheduleVideoUpdateModel } from '@server/models/video/schedule-video-update.js'import { VideoChannelActivityModel } from '@server/models/video/video-channel-activity.js'import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting.js'@@ -171,8 +172,12 @@ export class LocalVideoCreator {await setVideoTags({ video: this.video, tags: this.videoAttributes.tags, transaction })- const automaticTags = await new AutomaticTagger().buildVideoAutomaticTags({ video: this.video, transaction })- await setAndSaveVideoAutomaticTags({ video: this.video, automaticTags, transaction })+ const automaticTagsByAccount = await new AutomaticTagger().buildVideoAutomaticTags({+ serverAccount: await getServerAccount(),+ video: this.video,+ transaction+ })+ await setAndSaveVideoAutomaticTags({ video: this.video, automaticTagsByAccount, transaction })// Schedule an update in the future?if (this.videoAttributes.scheduleUpdate) {diff --git a/server/core/lib/notifier/shared/comment/comment-mention.ts b/server/core/lib/notifier/shared/comment/comment-mention.tsindex dfa9e24f1c..35e1d68c91 100644--- a/server/core/lib/notifier/shared/comment/comment-mention.ts+++ b/server/core/lib/notifier/shared/comment/comment-mention.ts@@ -3,8 +3,8 @@ import { tu } from '@server/helpers/i18n.js'import { logger } from '@server/helpers/logger.js'import { toSafeHtml } from '@server/helpers/markdown.js'import { WEBSERVER } from '@server/initializers/constants.js'+import { getServerAccount } from '@server/models/application/application.js'import { AccountBlocklistModel } from '@server/models/blocklist/account-blocklist.js'-import { getServerActor } from '@server/models/application/application.js'import { ServerBlocklistModel } from '@server/models/blocklist/server-blocklist.js'import { UserNotificationModel } from '@server/models/user/user-notification.js'import { UserModel } from '@server/models/user/user.js'@@ -50,7 +50,7 @@ export class CommentMention extends AbstractNotification<MCommentOwnerVideo, MUsif (this.users.length === 0) return- this.serverAccountId = (await getServerActor()).Account.id+ this.serverAccountId = (await getServerAccount()).idconst sourceAccounts = this.users.map(u => u.Account.id).concat([ this.serverAccountId ])diff --git a/server/core/lib/schedulers/watched-words-subscriptions-scheduler.ts b/server/core/lib/schedulers/watched-words-subscriptions-scheduler.tsnew file mode 100644index 0000000000..504e6908ee--- /dev/null+++ b/server/core/lib/schedulers/watched-words-subscriptions-scheduler.ts@@ -0,0 +1,179 @@+import {+ StreamSyncState,+ StreamSyncStateType,+ WatchedWordsSubscriptionAction,+ WatchedWordsSubscriptionActions+} from '@peertube/peertube-models'+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'+import { CONSTRAINTS_FIELDS, SCHEDULER_INTERVALS_MS } from '@server/initializers/constants.js'+import { sequelizeTypescript } from '@server/initializers/database.js'+import { fetchAndValidateWatchedWordsSubscriptionActions } from '@server/lib/watched-words-subscriptions.js'+import { WatchedWordsListModel } from '@server/models/watched-words/watched-words-list.js'+import { WatchedWordsSubscriptionModel } from '@server/models/watched-words/watched-words-subscription.js'+import { MWatchedWordsSubscription } from '@server/types/models/index.js'+import { createRebuildAutomaticTagsJob } from '../automatic-tags/automatic-tags.js'+import { AbstractScheduler } from './abstract-scheduler.js'++const lTags = loggerTagsFactory('schedulers')++export class WatchedWordsSubscriptionsScheduler extends AbstractScheduler {+ private static instance: AbstractScheduler++ protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.WATCHED_WORDS_SUBSCRIPTIONS_SYNC++ private constructor () {+ super({ randomRunOnEnable: true })+ }++ protected async internalExecute () {+ const subscriptions = await WatchedWordsSubscriptionModel.findAll<MWatchedWordsSubscription>()+ if (subscriptions.length === 0) return++ for (const subscription of subscriptions) {+ try {+ await this.syncSubscription(subscription)+ } catch (err) {+ await subscription.update({ state: StreamSyncState.FAILED })++ logger.error('Cannot synchronize watched words subscription.', {+ err,+ url: subscription.url,+ subscriptionId: subscription.id,+ ...lTags()+ })+ }+ }+ }++ private async syncSubscription (subscription: MWatchedWordsSubscription) {+ logger.info(`Synchronizing watched words subscription "${subscription.name}" (${subscription.url}).`, lTags())++ await subscription.update({ state: StreamSyncState.PROCESSING })++ const body: WatchedWordsSubscriptionActions = await fetchAndValidateWatchedWordsSubscriptionActions(+ subscription.url,+ subscription.lastActionCreatedAt+ )++ const actions = body.actions+ .map(action => ({ ...action, createdAtMs: Date.parse(action.createdAt) }))+ .filter(action => {+ if (Number.isNaN(action.createdAtMs)) return false++ if (!subscription.lastActionCreatedAt) return true++ return action.createdAtMs > subscription.lastActionCreatedAt.getTime()+ })+ .sort((a, b) => a.createdAtMs - b.createdAtMs)++ const state = await this.handleActions({ actions, subscription, listName: body.name })++ const lastActionCreatedAt = actions.length !== 0+ ? new Date(actions[actions.length - 1].createdAt)+ : subscription.lastActionCreatedAt++ await subscription.update({+ name: body.name,+ lastSyncAt: new Date(),+ lastActionCreatedAt,+ state+ })+ }++ private async handleActions (options: {+ actions: WatchedWordsSubscriptionAction[]+ subscription: MWatchedWordsSubscription+ listName: string+ }): Promise<StreamSyncStateType> {+ const { actions, subscription, listName } = options++ let shouldRebuildAutomaticTags = false++ await sequelizeTypescript.transaction(async transaction => {+ const existing = await WatchedWordsListModel.loadByListName({+ accountId: subscription.accountId,+ listName,+ transaction+ })++ let list = await WatchedWordsListModel.loadBySubscriptionId({+ accountId: subscription.accountId,+ watchedWordsSubscriptionId: subscription.id,+ transaction+ })++ if (existing && (!list || list.id !== existing.id)) {+ logger.error(+ 'Cannot create/update watched words list for subscription: a list with the same name already exists for this account',+ {+ accountId: subscription.accountId,+ subscriptionId: subscription.id,+ listName,+ existingListId: existing.id,+ ...lTags()+ }+ )+ return StreamSyncState.FAILED+ }++ if (actions.length === 0) {+ if (list && list.listName !== listName) {+ shouldRebuildAutomaticTags = true++ list.listName = listName+ await list.save({ transaction })+ }++ return StreamSyncState.SYNCED+ }++ shouldRebuildAutomaticTags = true++ if (!list) {+ list = await WatchedWordsListModel.createList({+ accountId: subscription.accountId,+ listName,+ watchedWordsSubscriptionId: subscription.id,+ words: [],+ transaction+ })+ }++ const wordsSet = new Set(list.words)++ for (const action of actions) {+ if (action.type === 'add') wordsSet.add(action.word)+ else wordsSet.delete(action.word)+ }++ if (wordsSet.size === 0) {+ return list.destroy({ transaction })+ }++ if (wordsSet.size > CONSTRAINTS_FIELDS.WATCHED_WORDS.WORDS.max) {+ logger.error('Cannot synchronize watched words subscription: too many items in watched words list.', {+ accountId: subscription.accountId,+ subscriptionId: subscription.id,+ listName,+ maxItems: CONSTRAINTS_FIELDS.WATCHED_WORDS.WORDS.max,+ itemsToStore: wordsSet.size,+ ...lTags()+ })++ return StreamSyncState.FAILED+ }++ await list.updateList({ listName, words: Array.from(wordsSet), transaction })+ })++ if (shouldRebuildAutomaticTags) {+ await createRebuildAutomaticTagsJob({ accountId: subscription.accountId })+ }++ return StreamSyncState.SYNCED+ }++ static get Instance () {+ return this.instance || (this.instance = new this())+ }+}diff --git a/server/core/lib/video-comment.ts b/server/core/lib/video-comment.tsindex 731089daa6..30069982e4 100644--- a/server/core/lib/video-comment.ts+++ b/server/core/lib/video-comment.ts@@ -2,6 +2,7 @@ import { AutomaticTagPolicy, ResultList, UserRight, VideoCommentPolicy, VideoComimport { logger } from '@server/helpers/logger.js'import { sequelizeTypescript } from '@server/initializers/database.js'import { AccountModel } from '@server/models/account/account.js'+import { getServerAccount } from '@server/models/application/application.js'import { AccountAutomaticTagPolicyModel } from '@server/models/automatic-tag/account-automatic-tag-policy.js'import express from 'express'import cloneDeep from 'lodash-es/cloneDeep.js'@@ -86,13 +87,19 @@ export async function createLocalVideoComment (options: {return sequelizeTypescript.transaction(async transaction => {const account = await AccountModel.load(user.Account.id, transaction)- const automaticTags = await new AutomaticTagger().buildCommentsAutomaticTags({+ const automaticTagsByAccount = await new AutomaticTagger().buildCommentsAutomaticTags({+ serverAccount: await getServerAccount(),ownerAccount: video.VideoChannel.Account,text,transaction})- const heldForReview = await shouldCommentBeHeldForReview({ user, video, automaticTags, transaction })+ const heldForReview = await shouldCommentBeHeldForReview({+ user,+ video,+ ownerAutomaticTags: automaticTagsByAccount[video.VideoChannel.accountId],+ transaction+ })const comment = await VideoCommentModel.create({text,@@ -108,7 +115,7 @@ export async function createLocalVideoComment (options: {const savedComment: MCommentOwnerVideoReply = await comment.save({ transaction })- await setAndSaveCommentAutomaticTags({ comment: savedComment, automaticTags, transaction })+ await setAndSaveCommentAutomaticTags({ comment: savedComment, automaticTagsByAccount, transaction })savedComment.InReplyToVideoComment = inReplyToCommentsavedComment.Video = video@@ -155,10 +162,10 @@ export function buildFormattedCommentTree (resultList: ResultList<MCommentFormatexport async function shouldCommentBeHeldForReview (options: {user: MUserAccountIdvideo: MVideoAccountLight- automaticTags: { name: string, accountId: number }[]+ ownerAutomaticTags: string[]transaction?: Transaction}) {- const { user, video, transaction, automaticTags } = options+ const { user, video, transaction, ownerAutomaticTags } = optionsif (video.isLocal() && user) {if (user.hasRight(UserRight.MANAGE_ANY_VIDEO_COMMENT)) return false@@ -168,16 +175,12 @@ export async function shouldCommentBeHeldForReview (options: {if (video.commentsPolicy === VideoCommentPolicy.REQUIRES_APPROVAL) return trueif (video.isLocal() !== true) return false- const ownerAccountTags = automaticTags- .filter(t => t.accountId === video.VideoChannel.accountId)- .map(t => t.name)-- if (ownerAccountTags.length === 0) return false+ if (!ownerAutomaticTags || ownerAutomaticTags.length === 0) return falsereturn AccountAutomaticTagPolicyModel.hasPolicyOnTags({accountId: video.VideoChannel.accountId,policy: AutomaticTagPolicy.REVIEW_COMMENT,- tags: ownerAccountTags,+ tags: ownerAutomaticTags,transaction})}diff --git a/server/core/lib/watched-words-subscriptions.ts b/server/core/lib/watched-words-subscriptions.tsnew file mode 100644index 0000000000..89eb25fd39--- /dev/null+++ b/server/core/lib/watched-words-subscriptions.ts@@ -0,0 +1,76 @@+import { WatchedWordsSubscriptionAction, WatchedWordsSubscriptionActions } from '@peertube/peertube-models'+import { doJSONRequest } from '@server/helpers/requests.js'+import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'++export async function fetchAndValidateWatchedWordsSubscriptionActions (+ url: string,+ startDate?: Date+): Promise<WatchedWordsSubscriptionActions> {+ const searchParams = startDate+ ? { startDate: startDate.toISOString() }+ : undefined++ const { body } = await doJSONRequest<WatchedWordsSubscriptionActions>(url, { searchParams })++ if (!body || typeof body !== 'object' || Array.isArray(body)) {+ throw new Error('Invalid watched words subscription payload: expected an object')+ }++ const name = body.name+ const rawActions = body.actions++ if (typeof name !== 'string' || name.trim().length === 0) {+ throw new Error('Invalid watched words subscription payload: name must be a non-empty string')+ }++ if (!Array.isArray(rawActions)) {+ throw new Error('Invalid watched words subscription payload: actions must be an array')+ }++ const maxActions = CONSTRAINTS_FIELDS.WATCHED_WORDS.WORDS.max * 5+ if (rawActions.length > maxActions) {+ throw new Error(+ `Invalid watched words subscription payload: actions length must not exceed ${maxActions}`+ )+ }++ const actions = rawActions.map((action, index) => validateWatchedWordsSubscriptionAction(action, index))++ return {+ name,+ actions+ }+}++// ---------------------------------------------------------------------------+// Private+// ---------------------------------------------------------------------------++function validateWatchedWordsSubscriptionAction (+ action: WatchedWordsSubscriptionActions['actions'][number],+ index: number+): WatchedWordsSubscriptionAction {+ if (!action || typeof action !== 'object' || Array.isArray(action)) {+ throw new Error(`Invalid watched words subscription action at index ${index}: expected object`)+ }++ const { type, word, createdAt } = action++ if (type !== 'add' && type !== 'remove') {+ throw new Error(`Invalid watched words subscription action at index ${index}: type must be add/remove`)+ }++ if (typeof word !== 'string' || word.trim().length === 0) {+ throw new Error(`Invalid watched words subscription action at index ${index}: word must be a non-empty string`)+ }++ if (typeof createdAt !== 'string' || Number.isNaN(Date.parse(createdAt))) {+ throw new Error(`Invalid watched words subscription action at index ${index}: createdAt must be a valid ISO date`)+ }++ return {+ type,+ word,+ createdAt+ }+}diff --git a/server/core/middlewares/validators/blocklist.ts b/server/core/middlewares/validators/blocklist.tsindex d06323a19f..2d583225ec 100644--- a/server/core/middlewares/validators/blocklist.ts+++ b/server/core/middlewares/validators/blocklist.ts@@ -156,7 +156,7 @@ export const addBlocklistSubscriptionValidator = [if (existingSubscription) {return res.fail({status: HttpStatusCode.CONFLICT_409,- message: req.t('Subscription with URL {{url}} already exists', { url: req.body.url })+ message: req.t('Subscription with URL {url} already exists', { url: req.body.url })})}diff --git a/server/core/middlewares/validators/index.ts b/server/core/middlewares/validators/index.tsindex 030c888d0d..8125639531 100644--- a/server/core/middlewares/validators/index.ts+++ b/server/core/middlewares/validators/index.ts@@ -23,6 +23,7 @@ export * from './sort.js'export * from './static.js'export * from './themes.js'export * from './webfinger.js'+export * from './watched-words-subscriptions.js'export * from './users/index.js'export * from './videos/index.js'export * from './runners/index.js'diff --git a/server/core/middlewares/validators/sort.ts b/server/core/middlewares/validators/sort.tsindex 68e7029394..c8188b3b1c 100644--- a/server/core/middlewares/validators/sort.ts+++ b/server/core/middlewares/validators/sort.ts@@ -33,6 +33,7 @@ export const videoPasswordsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDexport const changeOwnershipSortValidator = checkSortFactory(SORTABLE_COLUMNS.CHANGE_OWNERSHIP)export const watchedWordsListsSortValidator = checkSortFactory(SORTABLE_COLUMNS.WATCHED_WORDS_LISTS)+export const watchedWordsSubscriptionsSortValidator = checkSortFactory(SORTABLE_COLUMNS.WATCHED_WORDS_SUBSCRIPTIONS)export const accountsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNT_FOLLOWERS)export const videoChannelFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.CHANNEL_FOLLOWERS)diff --git a/server/core/middlewares/validators/watched-words-subscriptions.ts b/server/core/middlewares/validators/watched-words-subscriptions.tsnew file mode 100644index 0000000000..d8f28f1dd9--- /dev/null+++ b/server/core/middlewares/validators/watched-words-subscriptions.ts@@ -0,0 +1,60 @@+import { HttpStatusCode } from '@peertube/peertube-models'+import { Awaitable } from '@peertube/peertube-typescript-utils'+import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js'+import { isIdValid } from '@server/helpers/custom-validators/misc.js'+import { WatchedWordsSubscriptionModel } from '@server/models/watched-words/watched-words-subscription.js'+import { MAccountId } from '@server/types/models/index.js'+import express from 'express'+import { body, param } from 'express-validator'+import { areValidationErrors } from './shared/utils.js'++export function addWatchedWordsSubscriptionValidatorFactory (accountGetter: (res: express.Response) => Awaitable<MAccountId>) {+ return [+ body('url').custom(isUrlValid),++ async (req: express.Request, res: express.Response, next: express.NextFunction) => {+ if (areValidationErrors(req, res)) return++ const account = await accountGetter(res)++ const existingSubscription = await WatchedWordsSubscriptionModel.loadByUrl({+ accountId: account.id,+ url: req.body.url+ })++ if (existingSubscription) {+ return res.fail({+ status: HttpStatusCode.CONFLICT_409,+ message: req.t('Subscription with URL {url} already exists', { url: req.body.url })+ })+ }++ return next()+ }+ ]+}++export function deleteWatchedWordsSubscriptionValidatorFactory (accountGetter: (res: express.Response) => Awaitable<MAccountId>) {+ return [+ param('id')+ .custom(isIdValid),++ async (req: express.Request, res: express.Response, next: express.NextFunction) => {+ if (areValidationErrors(req, res)) return++ const account = await accountGetter(res)++ const subscription = await WatchedWordsSubscriptionModel.loadById(+req.params.id)+ if (!subscription || subscription.accountId !== account.id) {+ return res.fail({+ status: HttpStatusCode.NOT_FOUND_404,+ message: req.t('Watched words subscription not found')+ })+ }++ res.locals.watchedWordsSubscription = subscription++ return next()+ }+ ]+}diff --git a/server/core/middlewares/validators/watched-words.ts b/server/core/middlewares/validators/watched-words.tsindex be0261a122..18b2372958 100644--- a/server/core/middlewares/validators/watched-words.ts+++ b/server/core/middlewares/validators/watched-words.ts@@ -77,7 +77,7 @@ export function addWatchedWordsListValidatorFactory (accountGetter: (res: expresif (areValidationErrors(req, res)) returnconst listName = req.body.listName- if (!await checkListNameIsUnique({ accountId: (await accountGetter(res)).id, listName, res })) return+ if (!await checkListNameIsUnique({ accountId: (await accountGetter(res)).id, listName, req, res })) returnreturn next()}@@ -93,7 +93,7 @@ export function updateWatchedWordsListValidatorFactory (accountGetter: (res: expconst currentList = res.locals.watchedWordsListconst listName = req.body.listName- if (listName && !await checkListNameIsUnique({ accountId: (await accountGetter(res)).id, listName, currentList, res })) return+ if (listName && !await checkListNameIsUnique({ accountId: (await accountGetter(res)).id, listName, currentList, req, res })) returnreturn next()}@@ -107,16 +107,17 @@ export function updateWatchedWordsListValidatorFactory (accountGetter: (res: expasync function checkListNameIsUnique (options: {accountId: numberlistName: string+ req: express.Requestres: express.ResponsecurrentList?: MWatchedWordsList}) {- const { accountId, listName, currentList, res } = options+ const { accountId, listName, currentList, req, res } = optionsconst existing = await WatchedWordsListModel.loadByListName({ accountId, listName })if (existing && (currentList?.id !== existing.id)) {res.fail({status: HttpStatusCode.BAD_REQUEST_400,- message: `Watched words list with name ${listName} already exists`+ message: req.t(`Watched words list with name {listName} already exists`, { listName })})return falsediff --git a/server/core/models/application/application.ts b/server/core/models/application/application.tsindex 28501a4d03..c0d9726441 100644--- a/server/core/models/application/application.ts+++ b/server/core/models/application/application.ts@@ -25,6 +25,10 @@ export const getServerActor = memoizee(async function () {return actor}, { promise: true })+export function getServerAccount () {+ return getServerActor().then(actor => actor.Account)+}+type ConfigPart = PickDeep<typeof CONFIG, 'OBJECT_STORAGE.STREAMING_PLAYLISTS'>@DefaultScope(() => ({diff --git a/server/core/models/automatic-tag/comment-automatic-tag.ts b/server/core/models/automatic-tag/comment-automatic-tag.tsindex da856da944..314cf0d77d 100644--- a/server/core/models/automatic-tag/comment-automatic-tag.ts+++ b/server/core/models/automatic-tag/comment-automatic-tag.ts@@ -1,9 +1,10 @@+import { MCommentAutomaticTagWithTag } from '@server/types/models/index.js'+import { Op, Transaction } from 'sequelize'import { BelongsTo, Column, CreatedAt, ForeignKey, PrimaryKey, Table, UpdatedAt } from 'sequelize-typescript'import { AccountModel } from '../account/account.js'import { SequelizeModel } from '../shared/index.js'import { VideoCommentModel } from '../video/video-comment.js'import { AutomaticTagModel } from './automatic-tag.js'-import { Transaction } from 'sequelize'/*** Sequelize doesn't seem to support many to many relation using BelongsToMany with 3 tables@@ -11,7 +12,13 @@ import { Transaction } from 'sequelize'*/@Table({- tableName: 'commentAutomaticTag'+ tableName: 'commentAutomaticTag',+ indexes: [+ {+ unique: true,+ fields: [ 'commentId', 'accountId', 'automaticTagId' ]+ }+ ]})export class CommentAutomaticTagModel extends SequelizeModel<CommentAutomaticTagModel> {@CreatedAt@@ -71,4 +78,28 @@ export class CommentAutomaticTagModel extends SequelizeModel<CommentAutomaticTagtransaction})}++ static listByAccountIdsAndCommentId (options: {+ accountIds: number[]+ commentId: number+ transaction?: Transaction+ }) {+ const { accountIds, commentId, transaction } = options++ return CommentAutomaticTagModel.findAll<MCommentAutomaticTagWithTag>({+ where: {+ commentId,+ accountId: {+ [Op.in]: accountIds+ }+ },+ include: [+ {+ model: AutomaticTagModel,+ required: true+ }+ ],+ transaction+ })+ }}diff --git a/server/core/models/automatic-tag/video-automatic-tag.ts b/server/core/models/automatic-tag/video-automatic-tag.tsindex 7d93754efa..d856fe8177 100644--- a/server/core/models/automatic-tag/video-automatic-tag.ts+++ b/server/core/models/automatic-tag/video-automatic-tag.ts@@ -1,4 +1,5 @@-import { Transaction } from 'sequelize'+import { MVideoAutomaticTagWithTag } from '@server/types/models/automatic-tag/index.js'+import { Op, Transaction } from 'sequelize'import { BelongsTo, Column, CreatedAt, ForeignKey, PrimaryKey, Table, UpdatedAt } from 'sequelize-typescript'import { AccountModel } from '../account/account.js'import { SequelizeModel } from '../shared/index.js'@@ -11,7 +12,13 @@ import { AutomaticTagModel } from './automatic-tag.js'*/@Table({- tableName: 'videoAutomaticTag'+ tableName: 'videoAutomaticTag',+ indexes: [+ {+ unique: true,+ fields: [ 'videoId', 'accountId', 'automaticTagId' ]+ }+ ]})export class VideoAutomaticTagModel extends SequelizeModel<VideoAutomaticTagModel> {@CreatedAt@@ -59,15 +66,26 @@ export class VideoAutomaticTagModel extends SequelizeModel<VideoAutomaticTagMode})declare Video: Awaited<VideoModel>- static deleteAllOfAccountAndVideo (options: {- accountId: number+ static listByAccountIdsAndVideoId (options: {videoId: number- transaction: Transaction+ accountIds: number[]+ transaction?: Transaction}) {- const { accountId, videoId, transaction } = options+ const { accountIds, videoId, transaction } = options- return this.destroy({- where: { accountId, videoId },+ return this.findAll<MVideoAutomaticTagWithTag>({+ where: {+ videoId,+ accountId: {+ [Op.in]: accountIds+ }+ },+ include: [+ {+ model: AutomaticTagModel,+ required: true+ }+ ],transaction})}diff --git a/server/core/models/video/video-comment.ts b/server/core/models/video/video-comment.tsindex f7dcd1c455..6d02c78289 100644--- a/server/core/models/video/video-comment.ts+++ b/server/core/models/video/video-comment.ts@@ -43,6 +43,7 @@ import {MCommentOwner,MCommentOwnerVideoFeed,MCommentOwnerVideoReply,+ MCommentVideo,MVideo,MVideoImmutable} from '../../types/models/video/index.js'@@ -294,6 +295,17 @@ export class VideoCommentModel extends SequelizeModel<VideoCommentModel> {return VideoCommentModel.findOne(query)}+ static loadByIdWithVideo (id: number, transaction?: Transaction): Promise<MCommentVideo> {+ const query = {+ where: {+ id+ },+ transaction+ }++ return VideoCommentModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)+ }+static loadByIdAndPopulateVideoAndAccountAndReply (id: number, transaction?: Transaction): Promise<MCommentOwnerVideoReply> {const query = {where: {@@ -601,6 +613,30 @@ export class VideoCommentModel extends SequelizeModel<VideoCommentModel> {})}+ static async batchListIds (options: {+ lastId: number+ batchSize: number+ deleted: false+ videoOwnerId?: number+ }) {+ const { lastId, batchSize, videoOwnerId } = options++ const videoOwnerWhere = videoOwnerId+ ? '"videoChannel"."accountId" = :videoOwnerId AND '+ : ''++ const rows = await sequelizeTypescript.query<{ id: number }>(+ `SELECT "videoComment"."id" FROM "videoComment" ` ++ `INNER JOIN "video" ON "video"."id" = "videoComment"."videoId" ` ++ `INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ` ++ `WHERE ${videoOwnerWhere} "deletedAt" IS NULL AND "videoComment"."id" > :lastId ` ++ `ORDER BY "videoComment"."id" ASC LIMIT :batchSize`,+ { replacements: { lastId, batchSize, videoOwnerId }, type: QueryTypes.SELECT }+ )++ return rows.map(r => r.id)+ }+// ---------------------------------------------------------------------------static async getStats () {diff --git a/server/core/models/video/video.ts b/server/core/models/video/video.tsindex d287d57a4b..fe11fa9b42 100644--- a/server/core/models/video/video.ts+++ b/server/core/models/video/video.ts@@ -975,6 +975,20 @@ export class VideoModel extends SequelizeModel<VideoModel> {.then(rows => rows.map(r => r.id))}+ static async batchListIds (options: {+ lastId: number+ batchSize: number+ }) {+ const { lastId, batchSize } = options++ const rows = await this.sequelize.query<{ id: number }>(+ 'SELECT "video"."id" FROM "video" WHERE "video"."id" > :lastId ORDER BY "video"."id" ASC LIMIT :batchSize',+ { replacements: { lastId, batchSize }, type: QueryTypes.SELECT }+ )++ return rows.map(r => r.id)+ }+static listAllAndSharedByActorForOutbox (actorIdArg: number, start: number, count: number) {const actorId = forceNumber(actorIdArg)diff --git a/server/core/models/watched-words/watched-words-list.ts b/server/core/models/watched-words/watched-words-list.tsindex eac20d46c4..0e15d433e0 100644--- a/server/core/models/watched-words/watched-words-list.ts+++ b/server/core/models/watched-words/watched-words-list.ts@@ -8,6 +8,7 @@ import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, Uimport { LRU_CACHE, USER_EXPORT_MAX_ITEMS } from '../../initializers/constants.js'import { AccountModel } from '../account/account.js'import { SequelizeModel, getSort } from '../shared/index.js'+import { WatchedWordsSubscriptionModel } from './watched-words-subscription.js'@Table({tableName: 'watchedWordsList',@@ -48,6 +49,20 @@ export class WatchedWordsListModel extends SequelizeModel<WatchedWordsListModel>})declare Account: Awaited<AccountModel>+ @ForeignKey(() => WatchedWordsSubscriptionModel)+ @AllowNull(true)+ @Column+ declare watchedWordsSubscriptionId: number++ @BelongsTo(() => WatchedWordsSubscriptionModel, {+ foreignKey: {+ name: 'watchedWordsSubscriptionId',+ allowNull: true+ },+ onDelete: 'CASCADE'+ })+ declare WatchedWordsSubscription: Awaited<WatchedWordsSubscriptionModel>+// accountId => reg expressionsprivate static readonly regexCache = new LRUCache<number, { listName: string, regex: RegExp }[]>({max: LRU_CACHE.WATCHED_WORDS_REGEX.MAX_SIZE,@@ -70,16 +85,42 @@ export class WatchedWordsListModel extends SequelizeModel<WatchedWordsListModel>static loadByListName (options: {listName: stringaccountId: number+ watchedWordsSubscriptionId?: number | null+ transaction?: Transaction}) {- const { listName, accountId } = options+ const { listName, accountId, watchedWordsSubscriptionId, transaction } = optionsconst query = {- where: { listName, accountId }+ where: {+ listName,+ accountId,++ ...(watchedWordsSubscriptionId !== undefined+ ? { watchedWordsSubscriptionId }+ : {})+ },+ transaction}return this.findOne(query)}+ static loadBySubscriptionId (options: {+ accountId: number+ watchedWordsSubscriptionId: number+ transaction: Transaction+ }) {+ const { accountId, watchedWordsSubscriptionId, transaction } = options++ return this.findOne<MWatchedWordsList>({+ where: {+ accountId,+ watchedWordsSubscriptionId+ },+ transaction+ })+ }+// ---------------------------------------------------------------------------static listNamesOf (account: MAccountId) {@@ -101,15 +142,26 @@ export class WatchedWordsListModel extends SequelizeModel<WatchedWordsListModel>}) {const { accountId, start, count, sort } = options+ const countQuery = {+ where: { accountId }+ }+const query = {offset: start,limit: count,order: getSort(sort),+ include: [+ {+ model: WatchedWordsSubscriptionModel.unscoped(),+ required: false,+ attributes: [ 'url' ]+ }+ ],where: { accountId }}return Promise.all([- WatchedWordsListModel.count(query),+ WatchedWordsListModel.count(countQuery),WatchedWordsListModel.findAll(query)]).then(([ total, data ]) => ({ total, data }))}@@ -145,9 +197,9 @@ export class WatchedWordsListModel extends SequelizeModel<WatchedWordsListModel>const result = models.map(m => ({ listName: m.listName, regex: wordsToRegExp(m.words) }))- this.regexCache.set(accountId, result)+ WatchedWordsListModel.regexCache.set(accountId, result)- logger.debug('Will cache watched words regex', { accountId, result, tags: [ 'watched-words' ] })+ logger.debug('Will cache watched words regex', { accountId, listNames: result.map(r => r.listName), tags: [ 'watched-words' ] })return result}@@ -157,17 +209,39 @@ export class WatchedWordsListModel extends SequelizeModel<WatchedWordsListModel>listName: stringwords: string[]+ watchedWordsSubscriptionId?: number++ transaction?: Transaction}) {WatchedWordsListModel.regexCache.delete(options.accountId)- return super.create(options)+ return super.create<MWatchedWordsList>(options, { transaction: options.transaction })+ }++ static removeImportedBySubscription (options: {+ accountId: number+ watchedWordsSubscriptionId: number+ transaction?: Transaction+ }) {+ const { accountId, watchedWordsSubscriptionId, transaction } = options++ WatchedWordsListModel.regexCache.delete(accountId)++ return WatchedWordsListModel.destroy({+ where: {+ accountId,+ watchedWordsSubscriptionId+ },+ transaction+ })}updateList (options: {listName: stringwords?: string[]+ transaction?: Transaction}) {- const { listName, words } = options+ const { listName, words, transaction } = optionsif (words?.length === 0) {throw new Error('Cannot update watched words with an empty list')@@ -178,13 +252,15 @@ export class WatchedWordsListModel extends SequelizeModel<WatchedWordsListModel>WatchedWordsListModel.regexCache.delete(this.accountId)- return this.save()+ return this.save({ transaction })}- destroy () {+ destroy (options: {+ transaction?: Transaction+ } = {}) {WatchedWordsListModel.regexCache.delete(this.accountId)- return super.destroy()+ return super.destroy(options)}toFormattedJSON (): WatchedWordsList {@@ -192,6 +268,7 @@ export class WatchedWordsListModel extends SequelizeModel<WatchedWordsListModel>id: this.id,listName: this.listName,words: this.words,+ subscriptionUrl: this.WatchedWordsSubscription?.url ?? null,updatedAt: this.updatedAt,createdAt: this.createdAt}diff --git a/server/core/models/watched-words/watched-words-subscription.ts b/server/core/models/watched-words/watched-words-subscription.tsnew file mode 100644index 0000000000..15292b2cde--- /dev/null+++ b/server/core/models/watched-words/watched-words-subscription.ts@@ -0,0 +1,161 @@+import { StreamSyncState, type StreamSyncStateType, WatchedWordsSubscription } from '@peertube/peertube-models'+import { STREAM_SYNC_STATE } from '@server/initializers/constants.js'+import { MWatchedWordsSubscription } from '@server/types/models/index.js'+import { FindOptions, literal, Op } from 'sequelize'+import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, HasMany, Table, UpdatedAt } from 'sequelize-typescript'+import { AccountModel } from '../account/account.js'+import { getSort, searchAttribute, SequelizeModel } from '../shared/index.js'+import { WatchedWordsListModel } from './watched-words-list.js'++@Table({+ tableName: 'watchedWordsSubscription',+ indexes: [+ {+ fields: [ 'accountId', 'url' ],+ unique: true+ }+ ]+})+export class WatchedWordsSubscriptionModel extends SequelizeModel<WatchedWordsSubscriptionModel> {+ @CreatedAt+ declare createdAt: Date++ @UpdatedAt+ declare updatedAt: Date++ @AllowNull(false)+ @Column+ declare name: string++ @AllowNull(false)+ @Column+ declare url: string++ @Column+ declare lastSyncAt: Date | null++ @Column+ declare lastActionCreatedAt: Date | null++ @AllowNull(false)+ @Default(StreamSyncState.WAITING_FIRST_RUN)+ @Column+ declare state: StreamSyncStateType++ declare importedWordsCount: number++ @ForeignKey(() => AccountModel)+ @Column+ declare accountId: number++ @BelongsTo(() => AccountModel, {+ foreignKey: {+ name: 'accountId',+ allowNull: false+ },+ onDelete: 'CASCADE'+ })+ declare Account: Awaited<AccountModel>++ @HasMany(() => WatchedWordsListModel, {+ foreignKey: {+ name: 'watchedWordsSubscriptionId',+ allowNull: true+ },+ onDelete: 'CASCADE'+ })+ declare WatchedWordsLists: Awaited<WatchedWordsListModel[]>++ static loadById (id: number) {+ return WatchedWordsSubscriptionModel.findByPk<MWatchedWordsSubscription>(id)+ }++ static loadByUrl (options: {+ accountId: number+ url: string+ }) {+ return WatchedWordsSubscriptionModel.findOne<MWatchedWordsSubscription>({+ where: {+ url: options.url,+ accountId: options.accountId+ }+ })+ }++ static listForApi (parameters: {+ accountId: number+ start: number+ count: number+ sort: string+ search?: string+ }) {+ const { start, count, sort, search, accountId } = parameters++ const where = {+ accountId,++ ...(search+ ? {+ [Op.or]: [+ searchAttribute(search, 'name'),+ searchAttribute(search, 'url')+ ]+ }+ : {})+ }++ const findQuery: FindOptions = {+ attributes: {+ include: [+ [+ literal(+ 'COALESCE((' ++ 'SELECT SUM(cardinality("watchedWordsList"."words")) ' ++ 'FROM "watchedWordsList" ' ++ 'WHERE "watchedWordsList"."watchedWordsSubscriptionId" = "WatchedWordsSubscriptionModel"."id"' ++ '), 0)'+ ),+ 'importedWordsCount'+ ]+ ]+ },+ offset: start,+ limit: count,+ order: getSort(sort),+ where+ }++ const countQuery = { where }++ return Promise.all([+ WatchedWordsSubscriptionModel.count(countQuery),+ WatchedWordsSubscriptionModel.findAll<MWatchedWordsSubscription>(findQuery)+ ]).then(([ total, data ]) => ({ total, data }))+ }++ toFormattedSummaryJSON (this: MWatchedWordsSubscription) {+ return {+ id: this.id,+ name: this.name,+ url: this.url+ }+ }++ toFormattedJSON (this: MWatchedWordsSubscription): WatchedWordsSubscription {+ const importedWordsCount = Number(this.getDataValue('importedWordsCount') ?? 0)++ return {+ id: this.id,+ name: this.name,+ url: this.url,+ importedWordsCount,+ lastSyncAt: this.lastSyncAt,+ state: {+ id: this.state,+ label: STREAM_SYNC_STATE[this.state]+ },+ createdAt: this.createdAt,+ updatedAt: this.updatedAt+ }+ }+}diff --git a/server/core/types/express.d.ts b/server/core/types/express.d.tsindex 02a4860f55..d35389ebe9 100644--- a/server/core/types/express.d.ts+++ b/server/core/types/express.d.ts@@ -36,6 +36,7 @@ import {MVideoWithBlacklist,MVideoWithRights,MWatchedWordsList,+ MWatchedWordsSubscription,MBlocklistSubscription} from '@server/types/models/index.js'import { MOAuthToken, MOAuthTokenUser } from '@server/types/models/oauth/oauth-token.js'@@ -260,6 +261,7 @@ declare module 'express' {userExport?: MUserExportwatchedWordsList?: MWatchedWordsList+ watchedWordsSubscription?: MWatchedWordsSubscriptiontokenSession?: MOAuthTokendiff --git a/server/core/types/models/watched-words/index.ts b/server/core/types/models/watched-words/index.tsindex 822e0e3c26..e23946114e 100644--- a/server/core/types/models/watched-words/index.ts+++ b/server/core/types/models/watched-words/index.ts@@ -1 +1,2 @@export * from './watched-words-list.js'+export * from './watched-words-subscription.js'diff --git a/server/core/types/models/watched-words/watched-words-list.ts b/server/core/types/models/watched-words/watched-words-list.tsindex 1abac63ba0..88423f6ef8 100644--- a/server/core/types/models/watched-words/watched-words-list.ts+++ b/server/core/types/models/watched-words/watched-words-list.ts@@ -1,3 +1,3 @@import { WatchedWordsListModel } from '@server/models/watched-words/watched-words-list.js'-export type MWatchedWordsList = Omit<WatchedWordsListModel, 'Account'>+export type MWatchedWordsList = Omit<WatchedWordsListModel, 'Account' | 'WatchedWordsSubscription'>diff --git a/server/core/types/models/watched-words/watched-words-subscription.ts b/server/core/types/models/watched-words/watched-words-subscription.tsnew file mode 100644index 0000000000..de01360eec--- /dev/null+++ b/server/core/types/models/watched-words/watched-words-subscription.ts@@ -0,0 +1,3 @@+import { WatchedWordsSubscriptionModel } from '@server/models/watched-words/watched-words-subscription.js'++export type MWatchedWordsSubscription = Omit<WatchedWordsSubscriptionModel, 'Account' | 'WatchedWordsLists'>diff --git a/server/server.ts b/server/server.tsindex 2102d8bd00..dd730e3b1f 100644--- a/server/server.ts+++ b/server/server.ts@@ -151,6 +151,7 @@ import { RunnerJobWatchDogScheduler } from './core/lib/schedulers/runner-job-watimport { UpdateVideosScheduler } from './core/lib/schedulers/update-videos-scheduler.js'import { VideoStatsBufferScheduler } from './core/lib/schedulers/video-stats-buffer-scheduler.js'import { VideosRedundancyScheduler } from './core/lib/schedulers/videos-redundancy-scheduler.js'+import { WatchedWordsSubscriptionsScheduler } from './core/lib/schedulers/watched-words-subscriptions-scheduler.js'import { YoutubeDlUpdateScheduler } from './core/lib/schedulers/youtube-dl-update-scheduler.js'import { advertiseDoNotTrack } from './core/middlewares/dnt.js'import { apiFailMiddleware } from './core/middlewares/error.js'@@ -324,6 +325,7 @@ async function startApplication () {PeerTubeVersionCheckScheduler.Instance.enable()AutoFollowIndexInstances.Instance.enable()BlocklistSubscriptionsScheduler.Instance.enable()+ WatchedWordsSubscriptionsScheduler.Instance.enable()RemoveDanglingResumableUploadsScheduler.Instance.enable()VideoChannelSyncLatestScheduler.Instance.enable()VideoStatsBufferScheduler.Instance.enable()diff --git a/support/doc/api/embeds.md b/support/doc/api/embeds.mdindex 942d0e6cff..808e9eea8b 100644--- a/support/doc/api/embeds.md+++ b/support/doc/api/embeds.md@@ -112,7 +112,7 @@ Default: no subtitle selected and then tries to restore the last subtitle set by### autoplayTry to automatically play the video.-Most web browsers disable video autoplay if the user did not interact with the video. You can try to bypass this limitation by muting the video+Most web browsers disable video autoplay if the user did not interact with the video. You can try to bypass this limitation by muting the video.Value must be `0` or `1`.Default: `0`diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yamlindex 92bf03f759..a38b02d262 100644--- a/support/doc/api/openapi.yaml+++ b/support/doc/api/openapi.yaml@@ -8620,6 +8620,180 @@ paths:responses:'204':description: successful operation++ /api/v1/watched-words/accounts/{accountName}/subscriptions:+ get:+ tags:+ - Watched Words+ summary: List account watched words subscriptions+ security:+ - OAuth2: []+ parameters:+ - name: accountName+ in: path+ required: true+ description: account name to list watched words subscriptions+ schema:+ type: string+ - $ref: '#/components/parameters/start'+ - $ref: '#/components/parameters/count'+ - $ref: '#/components/parameters/sort'+ - $ref: '#/components/parameters/search'+ responses:+ '200':+ description: successful operation+ content:+ application/json:+ schema:+ type: object+ properties:+ total:+ type: integer+ example: 1+ data:+ type: array+ items:+ $ref: '#/components/schemas/WatchedWordsSubscription'+ post:+ tags:+ - Watched Words+ summary: Add account watched words subscription+ security:+ - OAuth2: []+ parameters:+ - name: accountName+ in: path+ required: true+ schema:+ type: string+ requestBody:+ content:+ application/json:+ schema:+ type: object+ properties:+ url:+ type: string+ format: url+ required:+ - url+ responses:+ '200':+ description: successful operation+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/WatchedWordsSubscription'+ '400':+ description: cannot fetch or parse watched words URL+ '409':+ description: subscription already exists or watched words list with fetched name already exists++ /api/v1/watched-words/accounts/{accountName}/subscriptions/{id}:+ delete:+ tags:+ - Watched Words+ summary: Delete account watched words subscription+ security:+ - OAuth2: []+ parameters:+ - name: accountName+ in: path+ required: true+ schema:+ type: string+ - name: id+ in: path+ required: true+ schema:+ type: integer+ responses:+ '204':+ description: successful operation+ '404':+ description: watched words subscription does not exist++ /api/v1/watched-words/server/subscriptions:+ get:+ tags:+ - Watched Words+ summary: List server watched words subscriptions+ security:+ - OAuth2:+ - admin+ - moderator+ parameters:+ - $ref: '#/components/parameters/start'+ - $ref: '#/components/parameters/count'+ - $ref: '#/components/parameters/sort'+ - $ref: '#/components/parameters/search'+ responses:+ '200':+ description: successful operation+ content:+ application/json:+ schema:+ type: object+ properties:+ total:+ type: integer+ example: 1+ data:+ type: array+ items:+ $ref: '#/components/schemas/WatchedWordsSubscription'+ post:+ tags:+ - Watched Words+ summary: Add server watched words subscription+ security:+ - OAuth2:+ - admin+ - moderator+ requestBody:+ content:+ application/json:+ schema:+ type: object+ properties:+ url:+ type: string+ format: url+ required:+ - url+ responses:+ '200':+ description: successful operation+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/WatchedWordsSubscription'+ '400':+ description: cannot fetch or parse watched words URL+ '409':+ description: subscription already exists or watched words list with fetched name already exists++ /api/v1/watched-words/server/subscriptions/{id}:+ delete:+ tags:+ - Watched Words+ summary: Delete server watched words subscription+ security:+ - OAuth2:+ - admin+ - moderator+ parameters:+ - name: id+ in: path+ required: true+ schema:+ type: integer+ responses:+ '204':+ description: successful operation+ '404':+ description: watched words subscription does not exist+/api/v1/client-config/update-language:post:tags:@@ -13957,6 +14131,48 @@ components:type: arrayitems:type: string+ subscriptionUrl:+ type: string+ format: url+ nullable: true+ updatedAt:+ type: string+ format: date-time+ example: 2021-05-04T08:01:01.502Z+ createdAt:+ type: string+ format: date-time+ example: 2021-05-04T08:01:01.502Z++ WatchedWordsSubscription:+ properties:+ id:+ $ref: '#/components/schemas/id'+ name:+ type: string+ url:+ type: string+ format: url+ importedWordsCount:+ type: integer+ minimum: 0+ lastSyncAt:+ type: string+ format: date-time+ nullable: true+ example: 2021-05-04T08:01:01.502Z+ state:+ type: object+ properties:+ id:+ type: integer+ enum:+ - 1+ - 2+ - 3+ - 4+ label:+ type: stringupdatedAt:type: stringformat: date-time