Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

feat(files): add a disabled indexing state and update scheduler/admin indexing workflows

johavenf7fc4f1cbb90
diff --git a/backend/src/applications/files/interfaces/indexing.interface.ts b/backend/src/applications/files/interfaces/indexing.interface.ts
index 37ed50e7..9f2552a4 100644
--- a/backend/src/applications/files/interfaces/indexing.interface.ts
+++ b/backend/src/applications/files/interfaces/indexing.interface.ts
@@ -1,7 +1,8 @@
export enum IndexingState {
RUNNING = 'running',
STOPPING = 'stopping',
- IDLE = 'idle'
+ IDLE = 'idle',
+ DISABLED = 'disabled'
}
export interface IndexingStatus {
diff --git a/backend/src/applications/files/services/files-content-indexer.service.ts b/backend/src/applications/files/services/files-content-indexer.service.ts
index 8c3ec87b..3d5f3247 100644
--- a/backend/src/applications/files/services/files-content-indexer.service.ts
+++ b/backend/src/applications/files/services/files-content-indexer.service.ts
@@ -22,9 +22,11 @@ import { FILE_REPOSITORY } from '../constants/operations'
import { Cache } from '../../../infrastructure/cache/services/cache.service'
import { IndexingState, IndexingStatus } from '../interfaces/indexing.interface'
import { escapePath } from '../../../common/functions'
+import { configuration } from '../../../configuration/config.environment'
@Injectable()
export class FilesContentIndexer {
+ public isEnabled = configuration.applications.files.contentIndexing.enabled
private readonly maxDocumentSize = 150 * 1_000_000
private readonly logger = new Logger(FilesContentIndexer.name)
private ocrManager: OCRManager | null = null
@@ -44,13 +46,13 @@ export class FilesContentIndexer {
}
async startIndexing(): Promise<boolean> {
- if (await this.isRunning()) return false
+ if (!this.isEnabled || (await this.isRunning())) return false
this.parseAndIndexAllFiles().catch((e) => this.logger.error({ tag: this.startIndexing.name, msg: `${e}` }))
return true
}
async stopIndexing(): Promise<boolean> {
- if (!(await this.isRunning())) return false
+ if (!this.isEnabled || !(await this.isRunning())) return false
await this.cache.set(CACHE_INDEXING_RUNNING_KEY, IndexingState.STOPPING, CACHE_INDEXING_RUNNING_TTL)
return true
}
@@ -58,7 +60,7 @@ export class FilesContentIndexer {
async status(): Promise<IndexingStatus> {
const status: IndexingStatus = {
indexesCount: 0,
- state: (await this.cache.get(CACHE_INDEXING_RUNNING_KEY)) ?? IndexingState.IDLE,
+ state: this.isEnabled ? ((await this.cache.get(CACHE_INDEXING_RUNNING_KEY)) ?? IndexingState.IDLE) : IndexingState.DISABLED,
lastFullRunAt: (await this.cache.get(CACHE_INDEXING_LAST_RUN_KEY)) ?? null,
lastPartialRunAt: (await this.cache.get(CACHE_INDEXING_EVENT_LAST_RUN_KEY)) ?? null
}
diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.ts
index 88d8b404..9967e62e 100644
--- a/backend/src/applications/files/services/files-scheduler.service.ts
+++ b/backend/src/applications/files/services/files-scheduler.service.ts
@@ -4,7 +4,6 @@ import { isNotNull, sql } from 'drizzle-orm'
import { unionAll } from 'drizzle-orm/mysql-core'
import fs from 'node:fs/promises'
import path from 'node:path'
-import { configuration } from '../../../configuration/config.environment'
import { Cache } from '../../../infrastructure/cache/services/cache.service'
import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'
import { DBSchema } from '../../../infrastructure/database/interfaces/database.interface'
@@ -38,12 +37,12 @@ export class FilesScheduler {
@Timeout(10_000)
async onStartup(): Promise<void> {
try {
- await this.filesContentIndexer.resetRunningState()
+ await this.resetContentIndexingState()
await this.cleanupInterruptedTasks()
await this.clearRecentFiles()
await this.updateQuotas()
} catch (e) {
- this.logger.error(e)
+ this.logger.error({ tag: this.onStartup.name, msg: `${e}` })
}
}
@@ -52,7 +51,7 @@ export class FilesScheduler {
try {
await this.indexContentFiles()
} catch (e) {
- this.logger.error(e)
+ this.logger.error({ tag: this.afterStartup.name, msg: `${e}` })
}
}
@@ -67,7 +66,7 @@ export class FilesScheduler {
} finally {
this.isQuotaUpdateEntriesIsRunning = false
}
- if (!configuration.applications.files.contentIndexing.enabled || (await this.filesContentIndexer.isRunning())) return
+ if (!this.filesContentIndexer.isEnabled || (await this.filesContentIndexer.isRunning())) return
try {
await this.filesContentIndexer.updateIndexEntries()
} catch (e) {
@@ -78,26 +77,6 @@ export class FilesScheduler {
}
}
- async cleanupInterruptedTasks(): Promise<void> {
- this.logger.log({ tag: this.cleanupInterruptedTasks.name, msg: `START` })
- try {
- let nb = 0
- const keys = await this.cache.keys(`${CACHE_TASK_PREFIX}-*`)
- for (const key of keys) {
- const task = await this.cache.get(key)
- if (task && task.status === FileTaskStatus.PENDING) {
- task.status = FileTaskStatus.ERROR
- task.result = 'Interrupted'
- nb++
- this.cache.set(key, task).catch((e: Error) => this.logger.error({ tag: this.cleanupInterruptedTasks.name, msg: `${e}` }))
- }
- }
- this.logger.log({ tag: this.cleanupInterruptedTasks.name, msg: `${nb} tasks cleaned : END` })
- } catch (e) {
- this.logger.error({ tag: this.cleanupInterruptedTasks.name, msg: `${e}` })
- }
- }
-
@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
async cleanupUserTaskFiles(): Promise<void> {
this.logger.log({ tag: this.cleanupUserTaskFiles.name, msg: `START` })
@@ -136,7 +115,6 @@ export class FilesScheduler {
@Cron(CronExpression.EVERY_8_HOURS)
async clearRecentFiles(): Promise<void> {
- this.logger.log({ tag: this.clearRecentFiles.name, msg: `START` })
const keepNumber = 100
let nbCleared = 0
try {
@@ -157,13 +135,13 @@ export class FilesScheduler {
} catch (e) {
this.logger.error({ tag: this.clearRecentFiles.name, msg: `${e}` })
}
- this.logger.log({ tag: this.clearRecentFiles.name, msg: `${nbCleared} records cleared - END` })
+ this.logger.log({ tag: this.clearRecentFiles.name, msg: `${nbCleared} records cleared` })
}
@Cron(CronExpression.EVERY_4_HOURS)
async indexContentFiles(): Promise<void> {
// Conditional loading of file content indexing
- if (!configuration.applications.files.contentIndexing.enabled) return
+ if (!this.filesContentIndexer.isEnabled) return
if (await this.filesContentIndexer.isRunning()) {
this.pendingFullSync = true
this.logger.warn({ tag: this.indexContentFiles.name, msg: `SKIP (already running) - deferred` })
@@ -239,4 +217,28 @@ export class FilesScheduler {
this.logger.log({ tag: this.updateQuotas.name, msg: 'Share External Paths - END' })
this.isQuotaUpdateIsRunning = false
}
+
+ private async cleanupInterruptedTasks(): Promise<void> {
+ try {
+ let nb = 0
+ const keys = await this.cache.keys(`${CACHE_TASK_PREFIX}-*`)
+ for (const key of keys) {
+ const task = await this.cache.get(key)
+ if (task && task.status === FileTaskStatus.PENDING) {
+ task.status = FileTaskStatus.ERROR
+ task.result = 'Interrupted'
+ nb++
+ this.cache.set(key, task).catch((e: Error) => this.logger.error({ tag: this.cleanupInterruptedTasks.name, msg: `${e}` }))
+ }
+ }
+ this.logger.log({ tag: this.cleanupInterruptedTasks.name, msg: `${nb} tasks cleaned` })
+ } catch (e) {
+ this.logger.error({ tag: this.cleanupInterruptedTasks.name, msg: `${e}` })
+ }
+ }
+
+ private async resetContentIndexingState(): Promise<void> {
+ await this.filesContentIndexer.resetRunningState()
+ this.logger.log({ tag: this.resetContentIndexingState.name, msg: `done` })
+ }
}
diff --git a/frontend/src/app/applications/admin/components/admin-tools.component.html b/frontend/src/app/applications/admin/components/admin-tools.component.html
index fb341a31..411b18be 100644
--- a/frontend/src/app/applications/admin/components/admin-tools.component.html
+++ b/frontend/src/app/applications/admin/components/admin-tools.component.html
@@ -14,13 +14,13 @@
<div class="col-5 col-sm-6 text-end">
@switch (indexingStatus.state) {
@case (IndexingState.RUNNING) {
- <span class="badge bg-success" l10nTranslate>Running</span>
+ <span class="badge bg-success" l10nTranslate>{{indexingStatus.state}}</span>
}
@case (IndexingState.STOPPING) {
- <span class="badge bg-warning" l10nTranslate>Stopping</span>
+ <span class="badge bg-warning" l10nTranslate>{{indexingStatus.state}}</span>
}
@default {
- <span class="badge bg-secondary" l10nTranslate>Idle</span>
+ <span class="badge bg-secondary" l10nTranslate>{{indexingStatus.state}}</span>
}
}
</div>
@@ -59,7 +59,7 @@
@if (indexingStatus.state === IndexingState.RUNNING) {
<button
(click)="stopIndexing()"
- class="btn btn-warning btn-sm text-nowrap"
+ class="btn btn-sm btn-warning btn-sm text-nowrap"
type="button">
<fa-icon [icon]="icons.faStop"></fa-icon>
<span class="ms-1" l10nTranslate>Stop</span>
@@ -67,8 +67,8 @@
} @else {
<button
(click)="startIndexing()"
- [disabled]="indexingStatus.state === IndexingState.STOPPING"
- class="btn btn-success btn-sm text-nowrap"
+ [disabled]="indexingStatus.state === IndexingState.STOPPING || indexingStatus.state === IndexingState.DISABLED"
+ class="btn btn-sm btn-success btn-sm text-nowrap"
type="button">
<fa-icon [icon]="icons.faPlay"></fa-icon>
<span class="ms-1" l10nTranslate>Start</span>
@@ -79,23 +79,24 @@
<div class="text-muted no-select fs-sm my-3 px-1 me-auto" l10nTranslate>
Delete all indexes.
</div>
- @if (confirmResetIndexing) {
- <button
- (click)="confirmResetIndexing = false"
- [disabled]="indexingStatus.state === IndexingState.RUNNING"
- class="btn btn-secondary btn-sm text-nowrap me-2"
- type="button">
- <span l10nTranslate>Cancel</span>
- </button>
- }
+ @if (confirmResetIndexing) {
<button
- (click)="dropIndexes()"
- class="btn {{confirmResetIndexing ? 'btn-danger' : 'btn-secondary'}} btn-sm text-nowrap"
- type="button"
- [disabled]="indexingStatus.state !== IndexingState.IDLE || indexingStatus.indexesCount === 0">
- <fa-icon [icon]="icons.faTrash"></fa-icon>
- <span class="ms-1" l10nTranslate>Remove</span>
+ (click)="confirmResetIndexing = false"
+ [disabled]="indexingStatus.state === IndexingState.RUNNING"
+ class="btn btn-sm btn-secondary btn-sm text-nowrap me-2"
+ type="button">
+ <span l10nTranslate>Cancel</span>
</button>
+ }
+ <button
+ (click)="dropIndexes()"
+ class="btn btn-sm {{confirmResetIndexing ? 'btn-danger' : 'btn-secondary'}} btn-sm text-nowrap"
+ type="button"
+ [disabled]="(indexingStatus.state !== IndexingState.IDLE && indexingStatus.state !== IndexingState.DISABLED) ||
+ indexingStatus.indexesCount === 0">
+ <fa-icon [icon]="icons.faTrash"></fa-icon>
+ <span class="ms-1" l10nTranslate>Remove</span>
+ </button>
</div>
</div>
</div>
diff --git a/frontend/src/i18n/de.json b/frontend/src/i18n/de.json
index 0a272d34..56e36fbd 100644
--- a/frontend/src/i18n/de.json
+++ b/frontend/src/i18n/de.json
@@ -650,9 +650,9 @@
"State": "Status",
"Stop": "Stoppen",
"Indexes": "Indizes",
- "Running": "Läuft",
- "Stopping": "Wird gestoppt",
- "Idle": "Leerlauf",
+ "running": "läuft",
+ "stopping": "wird gestoppt",
+ "idle": "leerlauf",
"Preferences": "Einstellungen",
"of": "von",
"Sidebar auto-hide enabled": "Automatisches Ausblenden der Seitenleiste aktiviert",
diff --git a/frontend/src/i18n/en.json b/frontend/src/i18n/en.json
index cd31cafd..ab10b02d 100644
--- a/frontend/src/i18n/en.json
+++ b/frontend/src/i18n/en.json
@@ -1,13 +1,4 @@
{
- "Actions": "Actions",
- "Cancel": "Cancel",
- "Delete all indexes.": "Delete all indexes.",
- "Full-text indexing": "Full-text indexing",
- "Idle": "Idle",
- "Indexes": "Indexes",
- "Last full run": "Last full run",
- "Last partial run": "Last partial run",
- "Never": "Never",
"no_task": "No task",
"one_task": "{{ nb }} task completed",
"nb_tasks": "{{ nb }} tasks completed",
@@ -24,16 +15,9 @@
"delete_one_file": "Do you want to permanently delete <b>{{arg}}</b> ?",
"trash_multiple_files": "Would you like to move <b>{{arg}} files</b> to trash ?",
"delete_multiple_files": "Do you want to permanently delete these <b>{{arg}} files</b> ?",
- "Remove": "Remove",
- "Running": "Running",
"scheduler_unit_hour": "hours",
"scheduler_unit_minute": "minutes",
"scheduler_unit_day": "days",
- "Start": "Start",
- "Start or stop indexing.": "Start or stop indexing.",
- "State": "State",
- "Stop": "Stop",
- "Stopping": "Stopping",
"no_selection": "no item selected",
"one_selection": "{{ nb }} item selected",
"nb_selections": "{{ nb }} items selected"
diff --git a/frontend/src/i18n/es.json b/frontend/src/i18n/es.json
index bd79ef23..bcef6b32 100644
--- a/frontend/src/i18n/es.json
+++ b/frontend/src/i18n/es.json
@@ -650,9 +650,9 @@
"State": "Estado",
"Stop": "Detener",
"Indexes": "Índices",
- "Running": "En ejecución",
- "Stopping": "Deteniendo",
- "Idle": "Inactivo",
+ "running": "en ejecución",
+ "stopping": "deteniendo",
+ "idle": "inactivo",
"Preferences": "Preferencias",
"of": "de",
"Sidebar auto-hide enabled": "Ocultación automática de la barra lateral activada",
diff --git a/frontend/src/i18n/fr.json b/frontend/src/i18n/fr.json
index 87635e65..2ec74a6f 100644
--- a/frontend/src/i18n/fr.json
+++ b/frontend/src/i18n/fr.json
@@ -650,9 +650,9 @@
"State": "État",
"Stop": "Arrêter",
"Indexes": "Indexes",
- "Running": "En cours",
- "Stopping": "Arrêt en cours",
- "Idle": "Inactif",
+ "running": "en cours",
+ "stopping": "arrêt en cours",
+ "idle": "inactif",
"Preferences": "Préférences",
"of": "de",
"Sidebar auto-hide enabled": "Masquage automatique de la barre latérale activé",
diff --git a/frontend/src/i18n/hi.json b/frontend/src/i18n/hi.json
index defc271e..89631373 100644
--- a/frontend/src/i18n/hi.json
+++ b/frontend/src/i18n/hi.json
@@ -650,9 +650,9 @@
"State": "स्थिति",
"Stop": "रोकें",
"Indexes": "इंडेक्स",
- "Running": "चल रहा है",
- "Stopping": "रोका जा रहा है",
- "Idle": "निष्क्रिय",
+ "running": "चल रहा है",
+ "stopping": "रोका जा रहा है",
+ "idle": "निष्क्रिय",
"Preferences": "वरीयताएँ",
"of": "का",
"Sidebar auto-hide enabled": "साइडबार स्वतः छिपाना सक्षम",
diff --git a/frontend/src/i18n/it.json b/frontend/src/i18n/it.json
index 7e51e264..1c4080bd 100644
--- a/frontend/src/i18n/it.json
+++ b/frontend/src/i18n/it.json
@@ -650,9 +650,9 @@
"State": "Stato",
"Stop": "Arresta",
"Indexes": "Indici",
- "Running": "In esecuzione",
- "Stopping": "In arresto",
- "Idle": "Inattivo",
+ "running": "in esecuzione",
+ "stopping": "in arresto",
+ "idle": "inattivo",
"Preferences": "Preferenze",
"of": "di",
"Sidebar auto-hide enabled": "Nascondimento automatico della barra laterale attivato",
diff --git a/frontend/src/i18n/ja.json b/frontend/src/i18n/ja.json
index b85403e2..07fc7820 100644
--- a/frontend/src/i18n/ja.json
+++ b/frontend/src/i18n/ja.json
@@ -650,9 +650,9 @@
"State": "状態",
"Stop": "停止",
"Indexes": "インデックス",
- "Running": "実行中",
- "Stopping": "停止中",
- "Idle": "待機中",
+ "running": "実行中",
+ "stopping": "停止中",
+ "idle": "待機中",
"Preferences": "環境設定",
"of": "の",
"Sidebar auto-hide enabled": "サイドバー自動非表示が有効",
diff --git a/frontend/src/i18n/ko.json b/frontend/src/i18n/ko.json
index 1c464505..0166c429 100644
--- a/frontend/src/i18n/ko.json
+++ b/frontend/src/i18n/ko.json
@@ -650,9 +650,9 @@
"State": "상태",
"Stop": "중지",
"Indexes": "인덱스",
- "Running": "실행 중",
- "Stopping": "중지 중",
- "Idle": "유휴",
+ "running": "실행 중",
+ "stopping": "중지 중",
+ "idle": "유휴",
"Preferences": "환경 설정",
"of": "의",
"Sidebar auto-hide enabled": "사이드바 자동 숨김 활성화",
diff --git a/frontend/src/i18n/nl.json b/frontend/src/i18n/nl.json
index 35fbab32..37799330 100644
--- a/frontend/src/i18n/nl.json
+++ b/frontend/src/i18n/nl.json
@@ -650,9 +650,9 @@
"State": "Status",
"Stop": "Stop",
"Indexes": "Indexen",
- "Running": "Actief",
- "Stopping": "Bezig met stoppen",
- "Idle": "Inactief",
+ "running": "actief",
+ "stopping": "bezig met stoppen",
+ "idle": "inactief",
"Preferences": "Voorkeuren",
"of": "van",
"Sidebar auto-hide enabled": "Zijbalk automatisch verbergen ingeschakeld",
diff --git a/frontend/src/i18n/pl.json b/frontend/src/i18n/pl.json
index 5daadf61..d6dc5272 100644
--- a/frontend/src/i18n/pl.json
+++ b/frontend/src/i18n/pl.json
@@ -650,9 +650,9 @@
"State": "Stan",
"Stop": "Stop",
"Indexes": "Indeksy",
- "Running": "W trakcie",
- "Stopping": "Zatrzymywanie",
- "Idle": "Bezczynny",
+ "running": "w trakcie",
+ "stopping": "zatrzymywanie",
+ "idle": "bezczynny",
"Preferences": "Preferencje",
"of": "z",
"Sidebar auto-hide enabled": "Włączono automatyczne ukrywanie panelu bocznego",
diff --git a/frontend/src/i18n/pt-BR.json b/frontend/src/i18n/pt-BR.json
index cdd0cc13..1cafe294 100644
--- a/frontend/src/i18n/pt-BR.json
+++ b/frontend/src/i18n/pt-BR.json
@@ -650,9 +650,9 @@
"State": "Estado",
"Stop": "Parar",
"Indexes": "Índices",
- "Running": "Em execução",
- "Stopping": "Parando",
- "Idle": "Inativo",
+ "running": "em execução",
+ "stopping": "parando",
+ "idle": "inativo",
"Preferences": "Preferências",
"of": "de",
"Sidebar auto-hide enabled": "Ocultação automática da barra lateral ativada",
diff --git a/frontend/src/i18n/pt.json b/frontend/src/i18n/pt.json
index 0a0f7431..88abcf29 100644
--- a/frontend/src/i18n/pt.json
+++ b/frontend/src/i18n/pt.json
@@ -650,9 +650,9 @@
"State": "Estado",
"Stop": "Parar",
"Indexes": "Índices",
- "Running": "Em execução",
- "Stopping": "A parar",
- "Idle": "Inativo",
+ "running": "em execução",
+ "stopping": "a parar",
+ "idle": "inativo",
"Preferences": "Preferências",
"of": "de",
"Sidebar auto-hide enabled": "Ocultação automática da barra lateral ativada",
diff --git a/frontend/src/i18n/ru.json b/frontend/src/i18n/ru.json
index 280d2ee2..1ad39c69 100644
--- a/frontend/src/i18n/ru.json
+++ b/frontend/src/i18n/ru.json
@@ -650,9 +650,9 @@
"State": "Состояние",
"Stop": "Остановить",
"Indexes": "Индексы",
- "Running": "Выполняется",
- "Stopping": "Останавливается",
- "Idle": "Ожидание",
+ "running": "выполняется",
+ "stopping": "останавливается",
+ "idle": "ожидание",
"Preferences": "Настройки",
"of": "из",
"Sidebar auto-hide enabled": "Автоскрытие боковой панели включено",
diff --git a/frontend/src/i18n/tr.json b/frontend/src/i18n/tr.json
index 4d480dfa..4b3ecd06 100644
--- a/frontend/src/i18n/tr.json
+++ b/frontend/src/i18n/tr.json
@@ -650,9 +650,9 @@
"State": "Durum",
"Stop": "Durdur",
"Indexes": "Dizinler",
- "Running": "Çalışıyor",
- "Stopping": "Durduruluyor",
- "Idle": "Boşta",
+ "running": "çalışıyor",
+ "stopping": "durduruluyor",
+ "idle": "boşta",
"Preferences": "Tercihler",
"of": "/",
"Sidebar auto-hide enabled": "Kenar çubuğu otomatik gizleme etkin",
diff --git a/frontend/src/i18n/zh.json b/frontend/src/i18n/zh.json
index c4df56f9..64754a24 100644
--- a/frontend/src/i18n/zh.json
+++ b/frontend/src/i18n/zh.json
@@ -650,9 +650,9 @@
"State": "状态",
"Stop": "停止",
"Indexes": "索引",
- "Running": "运行中",
- "Stopping": "停止中",
- "Idle": "空闲",
+ "running": "运行中",
+ "stopping": "停止中",
+ "idle": "空闲",
"Preferences": "偏好设置",
"of": "的",
"Sidebar auto-hide enabled": "侧边栏自动隐藏已启用",