Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
feat(files): add a disabled indexing state and update scheduler/admin indexing workflows
johavenf7fc4f1cbb90
diff --git a/backend/src/applications/files/interfaces/indexing.interface.ts b/backend/src/applications/files/interfaces/indexing.interface.tsindex 37ed50e7..9f2552a4 100644--- a/backend/src/applications/files/interfaces/indexing.interface.ts+++ b/backend/src/applications/files/interfaces/indexing.interface.ts@@ -1,7 +1,8 @@export enum IndexingState {RUNNING = 'running',STOPPING = 'stopping',- IDLE = 'idle'+ IDLE = 'idle',+ DISABLED = 'disabled'}export interface IndexingStatus {diff --git a/backend/src/applications/files/services/files-content-indexer.service.ts b/backend/src/applications/files/services/files-content-indexer.service.tsindex 8c3ec87b..3d5f3247 100644--- a/backend/src/applications/files/services/files-content-indexer.service.ts+++ b/backend/src/applications/files/services/files-content-indexer.service.ts@@ -22,9 +22,11 @@ import { FILE_REPOSITORY } from '../constants/operations'import { Cache } from '../../../infrastructure/cache/services/cache.service'import { IndexingState, IndexingStatus } from '../interfaces/indexing.interface'import { escapePath } from '../../../common/functions'+import { configuration } from '../../../configuration/config.environment'@Injectable()export class FilesContentIndexer {+ public isEnabled = configuration.applications.files.contentIndexing.enabledprivate readonly maxDocumentSize = 150 * 1_000_000private readonly logger = new Logger(FilesContentIndexer.name)private ocrManager: OCRManager | null = null@@ -44,13 +46,13 @@ export class FilesContentIndexer {}async startIndexing(): Promise<boolean> {- if (await this.isRunning()) return false+ if (!this.isEnabled || (await this.isRunning())) return falsethis.parseAndIndexAllFiles().catch((e) => this.logger.error({ tag: this.startIndexing.name, msg: `${e}` }))return true}async stopIndexing(): Promise<boolean> {- if (!(await this.isRunning())) return false+ if (!this.isEnabled || !(await this.isRunning())) return falseawait this.cache.set(CACHE_INDEXING_RUNNING_KEY, IndexingState.STOPPING, CACHE_INDEXING_RUNNING_TTL)return true}@@ -58,7 +60,7 @@ export class FilesContentIndexer {async status(): Promise<IndexingStatus> {const status: IndexingStatus = {indexesCount: 0,- state: (await this.cache.get(CACHE_INDEXING_RUNNING_KEY)) ?? IndexingState.IDLE,+ state: this.isEnabled ? ((await this.cache.get(CACHE_INDEXING_RUNNING_KEY)) ?? IndexingState.IDLE) : IndexingState.DISABLED,lastFullRunAt: (await this.cache.get(CACHE_INDEXING_LAST_RUN_KEY)) ?? null,lastPartialRunAt: (await this.cache.get(CACHE_INDEXING_EVENT_LAST_RUN_KEY)) ?? null}diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.tsindex 88d8b404..9967e62e 100644--- a/backend/src/applications/files/services/files-scheduler.service.ts+++ b/backend/src/applications/files/services/files-scheduler.service.ts@@ -4,7 +4,6 @@ import { isNotNull, sql } from 'drizzle-orm'import { unionAll } from 'drizzle-orm/mysql-core'import fs from 'node:fs/promises'import path from 'node:path'-import { configuration } from '../../../configuration/config.environment'import { Cache } from '../../../infrastructure/cache/services/cache.service'import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'import { DBSchema } from '../../../infrastructure/database/interfaces/database.interface'@@ -38,12 +37,12 @@ export class FilesScheduler {@Timeout(10_000)async onStartup(): Promise<void> {try {- await this.filesContentIndexer.resetRunningState()+ await this.resetContentIndexingState()await this.cleanupInterruptedTasks()await this.clearRecentFiles()await this.updateQuotas()} catch (e) {- this.logger.error(e)+ this.logger.error({ tag: this.onStartup.name, msg: `${e}` })}}@@ -52,7 +51,7 @@ export class FilesScheduler {try {await this.indexContentFiles()} catch (e) {- this.logger.error(e)+ this.logger.error({ tag: this.afterStartup.name, msg: `${e}` })}}@@ -67,7 +66,7 @@ export class FilesScheduler {} finally {this.isQuotaUpdateEntriesIsRunning = false}- if (!configuration.applications.files.contentIndexing.enabled || (await this.filesContentIndexer.isRunning())) return+ if (!this.filesContentIndexer.isEnabled || (await this.filesContentIndexer.isRunning())) returntry {await this.filesContentIndexer.updateIndexEntries()} catch (e) {@@ -78,26 +77,6 @@ export class FilesScheduler {}}- async cleanupInterruptedTasks(): Promise<void> {- this.logger.log({ tag: this.cleanupInterruptedTasks.name, msg: `START` })- try {- let nb = 0- const keys = await this.cache.keys(`${CACHE_TASK_PREFIX}-*`)- for (const key of keys) {- const task = await this.cache.get(key)- if (task && task.status === FileTaskStatus.PENDING) {- task.status = FileTaskStatus.ERROR- task.result = 'Interrupted'- nb++- this.cache.set(key, task).catch((e: Error) => this.logger.error({ tag: this.cleanupInterruptedTasks.name, msg: `${e}` }))- }- }- this.logger.log({ tag: this.cleanupInterruptedTasks.name, msg: `${nb} tasks cleaned : END` })- } catch (e) {- this.logger.error({ tag: this.cleanupInterruptedTasks.name, msg: `${e}` })- }- }-@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)async cleanupUserTaskFiles(): Promise<void> {this.logger.log({ tag: this.cleanupUserTaskFiles.name, msg: `START` })@@ -136,7 +115,6 @@ export class FilesScheduler {@Cron(CronExpression.EVERY_8_HOURS)async clearRecentFiles(): Promise<void> {- this.logger.log({ tag: this.clearRecentFiles.name, msg: `START` })const keepNumber = 100let nbCleared = 0try {@@ -157,13 +135,13 @@ export class FilesScheduler {} catch (e) {this.logger.error({ tag: this.clearRecentFiles.name, msg: `${e}` })}- this.logger.log({ tag: this.clearRecentFiles.name, msg: `${nbCleared} records cleared - END` })+ this.logger.log({ tag: this.clearRecentFiles.name, msg: `${nbCleared} records cleared` })}@Cron(CronExpression.EVERY_4_HOURS)async indexContentFiles(): Promise<void> {// Conditional loading of file content indexing- if (!configuration.applications.files.contentIndexing.enabled) return+ if (!this.filesContentIndexer.isEnabled) returnif (await this.filesContentIndexer.isRunning()) {this.pendingFullSync = truethis.logger.warn({ tag: this.indexContentFiles.name, msg: `SKIP (already running) - deferred` })@@ -239,4 +217,28 @@ export class FilesScheduler {this.logger.log({ tag: this.updateQuotas.name, msg: 'Share External Paths - END' })this.isQuotaUpdateIsRunning = false}++ private async cleanupInterruptedTasks(): Promise<void> {+ try {+ let nb = 0+ const keys = await this.cache.keys(`${CACHE_TASK_PREFIX}-*`)+ for (const key of keys) {+ const task = await this.cache.get(key)+ if (task && task.status === FileTaskStatus.PENDING) {+ task.status = FileTaskStatus.ERROR+ task.result = 'Interrupted'+ nb+++ this.cache.set(key, task).catch((e: Error) => this.logger.error({ tag: this.cleanupInterruptedTasks.name, msg: `${e}` }))+ }+ }+ this.logger.log({ tag: this.cleanupInterruptedTasks.name, msg: `${nb} tasks cleaned` })+ } catch (e) {+ this.logger.error({ tag: this.cleanupInterruptedTasks.name, msg: `${e}` })+ }+ }++ private async resetContentIndexingState(): Promise<void> {+ await this.filesContentIndexer.resetRunningState()+ this.logger.log({ tag: this.resetContentIndexingState.name, msg: `done` })+ }}diff --git a/frontend/src/app/applications/admin/components/admin-tools.component.html b/frontend/src/app/applications/admin/components/admin-tools.component.htmlindex fb341a31..411b18be 100644--- a/frontend/src/app/applications/admin/components/admin-tools.component.html+++ b/frontend/src/app/applications/admin/components/admin-tools.component.html@@ -14,13 +14,13 @@<div class="col-5 col-sm-6 text-end">@switch (indexingStatus.state) {@case (IndexingState.RUNNING) {- <span class="badge bg-success" l10nTranslate>Running</span>+ <span class="badge bg-success" l10nTranslate>{{indexingStatus.state}}</span>}@case (IndexingState.STOPPING) {- <span class="badge bg-warning" l10nTranslate>Stopping</span>+ <span class="badge bg-warning" l10nTranslate>{{indexingStatus.state}}</span>}@default {- <span class="badge bg-secondary" l10nTranslate>Idle</span>+ <span class="badge bg-secondary" l10nTranslate>{{indexingStatus.state}}</span>}}</div>@@ -59,7 +59,7 @@@if (indexingStatus.state === IndexingState.RUNNING) {<button(click)="stopIndexing()"- class="btn btn-warning btn-sm text-nowrap"+ class="btn btn-sm btn-warning btn-sm text-nowrap"type="button"><fa-icon [icon]="icons.faStop"></fa-icon><span class="ms-1" l10nTranslate>Stop</span>@@ -67,8 +67,8 @@} @else {<button(click)="startIndexing()"- [disabled]="indexingStatus.state === IndexingState.STOPPING"- class="btn btn-success btn-sm text-nowrap"+ [disabled]="indexingStatus.state === IndexingState.STOPPING || indexingStatus.state === IndexingState.DISABLED"+ class="btn btn-sm btn-success btn-sm text-nowrap"type="button"><fa-icon [icon]="icons.faPlay"></fa-icon><span class="ms-1" l10nTranslate>Start</span>@@ -79,23 +79,24 @@<div class="text-muted no-select fs-sm my-3 px-1 me-auto" l10nTranslate>Delete all indexes.</div>- @if (confirmResetIndexing) {- <button- (click)="confirmResetIndexing = false"- [disabled]="indexingStatus.state === IndexingState.RUNNING"- class="btn btn-secondary btn-sm text-nowrap me-2"- type="button">- <span l10nTranslate>Cancel</span>- </button>- }+ @if (confirmResetIndexing) {<button- (click)="dropIndexes()"- class="btn {{confirmResetIndexing ? 'btn-danger' : 'btn-secondary'}} btn-sm text-nowrap"- type="button"- [disabled]="indexingStatus.state !== IndexingState.IDLE || indexingStatus.indexesCount === 0">- <fa-icon [icon]="icons.faTrash"></fa-icon>- <span class="ms-1" l10nTranslate>Remove</span>+ (click)="confirmResetIndexing = false"+ [disabled]="indexingStatus.state === IndexingState.RUNNING"+ class="btn btn-sm btn-secondary btn-sm text-nowrap me-2"+ type="button">+ <span l10nTranslate>Cancel</span></button>+ }+ <button+ (click)="dropIndexes()"+ class="btn btn-sm {{confirmResetIndexing ? 'btn-danger' : 'btn-secondary'}} btn-sm text-nowrap"+ type="button"+ [disabled]="(indexingStatus.state !== IndexingState.IDLE && indexingStatus.state !== IndexingState.DISABLED) ||+ indexingStatus.indexesCount === 0">+ <fa-icon [icon]="icons.faTrash"></fa-icon>+ <span class="ms-1" l10nTranslate>Remove</span>+ </button></div></div></div>diff --git a/frontend/src/i18n/de.json b/frontend/src/i18n/de.jsonindex 0a272d34..56e36fbd 100644--- a/frontend/src/i18n/de.json+++ b/frontend/src/i18n/de.json@@ -650,9 +650,9 @@"State": "Status","Stop": "Stoppen","Indexes": "Indizes",- "Running": "Läuft",- "Stopping": "Wird gestoppt",- "Idle": "Leerlauf",+ "running": "läuft",+ "stopping": "wird gestoppt",+ "idle": "leerlauf","Preferences": "Einstellungen","of": "von","Sidebar auto-hide enabled": "Automatisches Ausblenden der Seitenleiste aktiviert",diff --git a/frontend/src/i18n/en.json b/frontend/src/i18n/en.jsonindex cd31cafd..ab10b02d 100644--- a/frontend/src/i18n/en.json+++ b/frontend/src/i18n/en.json@@ -1,13 +1,4 @@{- "Actions": "Actions",- "Cancel": "Cancel",- "Delete all indexes.": "Delete all indexes.",- "Full-text indexing": "Full-text indexing",- "Idle": "Idle",- "Indexes": "Indexes",- "Last full run": "Last full run",- "Last partial run": "Last partial run",- "Never": "Never","no_task": "No task","one_task": "{{ nb }} task completed","nb_tasks": "{{ nb }} tasks completed",@@ -24,16 +15,9 @@"delete_one_file": "Do you want to permanently delete <b>{{arg}}</b> ?","trash_multiple_files": "Would you like to move <b>{{arg}} files</b> to trash ?","delete_multiple_files": "Do you want to permanently delete these <b>{{arg}} files</b> ?",- "Remove": "Remove",- "Running": "Running","scheduler_unit_hour": "hours","scheduler_unit_minute": "minutes","scheduler_unit_day": "days",- "Start": "Start",- "Start or stop indexing.": "Start or stop indexing.",- "State": "State",- "Stop": "Stop",- "Stopping": "Stopping","no_selection": "no item selected","one_selection": "{{ nb }} item selected","nb_selections": "{{ nb }} items selected"diff --git a/frontend/src/i18n/es.json b/frontend/src/i18n/es.jsonindex bd79ef23..bcef6b32 100644--- a/frontend/src/i18n/es.json+++ b/frontend/src/i18n/es.json@@ -650,9 +650,9 @@"State": "Estado","Stop": "Detener","Indexes": "Índices",- "Running": "En ejecución",- "Stopping": "Deteniendo",- "Idle": "Inactivo",+ "running": "en ejecución",+ "stopping": "deteniendo",+ "idle": "inactivo","Preferences": "Preferencias","of": "de","Sidebar auto-hide enabled": "Ocultación automática de la barra lateral activada",diff --git a/frontend/src/i18n/fr.json b/frontend/src/i18n/fr.jsonindex 87635e65..2ec74a6f 100644--- a/frontend/src/i18n/fr.json+++ b/frontend/src/i18n/fr.json@@ -650,9 +650,9 @@"State": "État","Stop": "Arrêter","Indexes": "Indexes",- "Running": "En cours",- "Stopping": "Arrêt en cours",- "Idle": "Inactif",+ "running": "en cours",+ "stopping": "arrêt en cours",+ "idle": "inactif","Preferences": "Préférences","of": "de","Sidebar auto-hide enabled": "Masquage automatique de la barre latérale activé",diff --git a/frontend/src/i18n/hi.json b/frontend/src/i18n/hi.jsonindex defc271e..89631373 100644--- a/frontend/src/i18n/hi.json+++ b/frontend/src/i18n/hi.json@@ -650,9 +650,9 @@"State": "स्थिति","Stop": "रोकें","Indexes": "इंडेक्स",- "Running": "चल रहा है",- "Stopping": "रोका जा रहा है",- "Idle": "निष्क्रिय",+ "running": "चल रहा है",+ "stopping": "रोका जा रहा है",+ "idle": "निष्क्रिय","Preferences": "वरीयताएँ","of": "का","Sidebar auto-hide enabled": "साइडबार स्वतः छिपाना सक्षम",diff --git a/frontend/src/i18n/it.json b/frontend/src/i18n/it.jsonindex 7e51e264..1c4080bd 100644--- a/frontend/src/i18n/it.json+++ b/frontend/src/i18n/it.json@@ -650,9 +650,9 @@"State": "Stato","Stop": "Arresta","Indexes": "Indici",- "Running": "In esecuzione",- "Stopping": "In arresto",- "Idle": "Inattivo",+ "running": "in esecuzione",+ "stopping": "in arresto",+ "idle": "inattivo","Preferences": "Preferenze","of": "di","Sidebar auto-hide enabled": "Nascondimento automatico della barra laterale attivato",diff --git a/frontend/src/i18n/ja.json b/frontend/src/i18n/ja.jsonindex b85403e2..07fc7820 100644--- a/frontend/src/i18n/ja.json+++ b/frontend/src/i18n/ja.json@@ -650,9 +650,9 @@"State": "状態","Stop": "停止","Indexes": "インデックス",- "Running": "実行中",- "Stopping": "停止中",- "Idle": "待機中",+ "running": "実行中",+ "stopping": "停止中",+ "idle": "待機中","Preferences": "環境設定","of": "の","Sidebar auto-hide enabled": "サイドバー自動非表示が有効",diff --git a/frontend/src/i18n/ko.json b/frontend/src/i18n/ko.jsonindex 1c464505..0166c429 100644--- a/frontend/src/i18n/ko.json+++ b/frontend/src/i18n/ko.json@@ -650,9 +650,9 @@"State": "상태","Stop": "중지","Indexes": "인덱스",- "Running": "실행 중",- "Stopping": "중지 중",- "Idle": "유휴",+ "running": "실행 중",+ "stopping": "중지 중",+ "idle": "유휴","Preferences": "환경 설정","of": "의","Sidebar auto-hide enabled": "사이드바 자동 숨김 활성화",diff --git a/frontend/src/i18n/nl.json b/frontend/src/i18n/nl.jsonindex 35fbab32..37799330 100644--- a/frontend/src/i18n/nl.json+++ b/frontend/src/i18n/nl.json@@ -650,9 +650,9 @@"State": "Status","Stop": "Stop","Indexes": "Indexen",- "Running": "Actief",- "Stopping": "Bezig met stoppen",- "Idle": "Inactief",+ "running": "actief",+ "stopping": "bezig met stoppen",+ "idle": "inactief","Preferences": "Voorkeuren","of": "van","Sidebar auto-hide enabled": "Zijbalk automatisch verbergen ingeschakeld",diff --git a/frontend/src/i18n/pl.json b/frontend/src/i18n/pl.jsonindex 5daadf61..d6dc5272 100644--- a/frontend/src/i18n/pl.json+++ b/frontend/src/i18n/pl.json@@ -650,9 +650,9 @@"State": "Stan","Stop": "Stop","Indexes": "Indeksy",- "Running": "W trakcie",- "Stopping": "Zatrzymywanie",- "Idle": "Bezczynny",+ "running": "w trakcie",+ "stopping": "zatrzymywanie",+ "idle": "bezczynny","Preferences": "Preferencje","of": "z","Sidebar auto-hide enabled": "Włączono automatyczne ukrywanie panelu bocznego",diff --git a/frontend/src/i18n/pt-BR.json b/frontend/src/i18n/pt-BR.jsonindex cdd0cc13..1cafe294 100644--- a/frontend/src/i18n/pt-BR.json+++ b/frontend/src/i18n/pt-BR.json@@ -650,9 +650,9 @@"State": "Estado","Stop": "Parar","Indexes": "Índices",- "Running": "Em execução",- "Stopping": "Parando",- "Idle": "Inativo",+ "running": "em execução",+ "stopping": "parando",+ "idle": "inativo","Preferences": "Preferências","of": "de","Sidebar auto-hide enabled": "Ocultação automática da barra lateral ativada",diff --git a/frontend/src/i18n/pt.json b/frontend/src/i18n/pt.jsonindex 0a0f7431..88abcf29 100644--- a/frontend/src/i18n/pt.json+++ b/frontend/src/i18n/pt.json@@ -650,9 +650,9 @@"State": "Estado","Stop": "Parar","Indexes": "Índices",- "Running": "Em execução",- "Stopping": "A parar",- "Idle": "Inativo",+ "running": "em execução",+ "stopping": "a parar",+ "idle": "inativo","Preferences": "Preferências","of": "de","Sidebar auto-hide enabled": "Ocultação automática da barra lateral ativada",diff --git a/frontend/src/i18n/ru.json b/frontend/src/i18n/ru.jsonindex 280d2ee2..1ad39c69 100644--- a/frontend/src/i18n/ru.json+++ b/frontend/src/i18n/ru.json@@ -650,9 +650,9 @@"State": "Состояние","Stop": "Остановить","Indexes": "Индексы",- "Running": "Выполняется",- "Stopping": "Останавливается",- "Idle": "Ожидание",+ "running": "выполняется",+ "stopping": "останавливается",+ "idle": "ожидание","Preferences": "Настройки","of": "из","Sidebar auto-hide enabled": "Автоскрытие боковой панели включено",diff --git a/frontend/src/i18n/tr.json b/frontend/src/i18n/tr.jsonindex 4d480dfa..4b3ecd06 100644--- a/frontend/src/i18n/tr.json+++ b/frontend/src/i18n/tr.json@@ -650,9 +650,9 @@"State": "Durum","Stop": "Durdur","Indexes": "Dizinler",- "Running": "Çalışıyor",- "Stopping": "Durduruluyor",- "Idle": "Boşta",+ "running": "çalışıyor",+ "stopping": "durduruluyor",+ "idle": "boşta","Preferences": "Tercihler","of": "/","Sidebar auto-hide enabled": "Kenar çubuğu otomatik gizleme etkin",diff --git a/frontend/src/i18n/zh.json b/frontend/src/i18n/zh.jsonindex c4df56f9..64754a24 100644--- a/frontend/src/i18n/zh.json+++ b/frontend/src/i18n/zh.json@@ -650,9 +650,9 @@"State": "状态","Stop": "停止","Indexes": "索引",- "Running": "运行中",- "Stopping": "停止中",- "Idle": "空闲",+ "running": "运行中",+ "stopping": "停止中",+ "idle": "空闲","Preferences": "偏好设置","of": "的","Sidebar auto-hide enabled": "侧边栏自动隐藏已启用",