Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

feat(files): add config toggles to enable/disable file indexing and full-text search

johavenfde7c58cbb94
diff --git a/backend/src/applications/files/files.config.ts b/backend/src/applications/files/files.config.ts
index 346244e4..e4deaf12 100644
--- a/backend/src/applications/files/files.config.ts
+++ b/backend/src/applications/files/files.config.ts
@@ -44,6 +44,9 @@ export class FilesConfig {
@IsInt()
maxUploadSize: number = 5368709120 // 5 GB
+ @IsBoolean()
+ contentIndexing: boolean = true
+
@IsBoolean()
showHiddenFiles: boolean = false
diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.ts
index ea4aa304..fd0203a9 100644
--- a/backend/src/applications/files/services/files-manager.service.ts
+++ b/backend/src/applications/files/services/files-manager.service.ts
@@ -339,7 +339,7 @@ export class FilesManager {
}
}
- // check lock conflicts on source & destination
+ // check lock conflicts on source and destination
let recursive: boolean
let depth: LOCK_DEPTH
if (dav?.depth) {
@@ -362,7 +362,7 @@ export class FilesManager {
await this.delete(user, dstSpace)
}
- // send to task watcher
+ // send it to task watcher
if (srcSpace.task?.cacheKey) {
if (!isDir) srcSpace.task.props.totalSize = await fileSize(srcSpace.realPath)
FileTaskEvent.emit('startWatch', srcSpace, isMove ? FILE_OPERATION.MOVE : FILE_OPERATION.COPY, dstSpace.realPath)
diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.ts
index d67ddc64..a4c94e05 100644
--- a/backend/src/applications/files/services/files-scheduler.service.ts
+++ b/backend/src/applications/files/services/files-scheduler.service.ts
@@ -4,11 +4,13 @@
* See the LICENSE file for licensing details
*/
-import { Inject, Injectable, Logger } from '@nestjs/common'
-import { Cron, CronExpression, Timeout } from '@nestjs/schedule'
+import { Inject, Injectable, Logger, OnModuleInit } from '@nestjs/common'
+import { Cron, CronExpression, SchedulerRegistry, Timeout } from '@nestjs/schedule'
+import { CronJob } from 'cron'
import { sql } from 'drizzle-orm'
import fs from 'node:fs/promises'
import path from 'node:path'
+import { configuration } from '../../../configuration/config.environment'
import { Cache } from '../../../infrastructure/cache/services/cache.service'
import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'
import { DBSchema } from '../../../infrastructure/database/interfaces/database.interface'
@@ -23,15 +25,26 @@ import { FilesContentManager } from './files-content-manager.service'
import { FilesTasksManager } from './files-tasks-manager.service'
@Injectable()
-export class FilesScheduler {
+export class FilesScheduler implements OnModuleInit {
private readonly logger = new Logger(FilesScheduler.name)
constructor(
@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema,
+ private readonly schedulerRegistry: SchedulerRegistry,
private readonly cache: Cache,
private readonly filesContentManager: FilesContentManager
) {}
+ onModuleInit() {
+ if (!configuration.applications.files.contentIndexing) return
+ // Conditional loading of file content indexing
+ const job = new CronJob(CronExpression.EVERY_4_HOURS, () => this.indexContentFiles())
+ this.schedulerRegistry.addCronJob('indexContentFiles', job)
+ job.start()
+ const timeout = setTimeout(() => this.indexContentFiles(), 60_000)
+ this.schedulerRegistry.addTimeout('bootstrapIndexContentFiles', timeout)
+ }
+
@Timeout(30000)
async cleanupInterruptedTasks(): Promise<void> {
this.logger.log(`${this.cleanupInterruptedTasks.name} - START`)
@@ -116,8 +129,6 @@ export class FilesScheduler {
this.logger.log(`${this.clearRecentFiles.name} - ${nbCleared} records cleared - END`)
}
- @Timeout(60000)
- @Cron(CronExpression.EVERY_4_HOURS)
async indexContentFiles(): Promise<void> {
this.logger.log(`${this.indexContentFiles.name} - START`)
await this.filesContentManager.parseAndIndexAllFiles()
diff --git a/backend/src/applications/files/services/files-search-manager.service.ts b/backend/src/applications/files/services/files-search-manager.service.ts
index 28dbe998..904093b5 100644
--- a/backend/src/applications/files/services/files-search-manager.service.ts
+++ b/backend/src/applications/files/services/files-search-manager.service.ts
@@ -8,6 +8,7 @@ import { HttpException, HttpStatus, Injectable, Logger } from '@nestjs/common'
import fs from 'fs/promises'
import { Stats } from 'node:fs'
import path from 'node:path'
+import { configuration } from '../../../configuration/config.environment'
import { SharesQueries } from '../../shares/services/shares-queries.service'
import { SpacesQueries } from '../../spaces/services/spaces-queries.service'
import { UserModel } from '../../users/models/user.model'
@@ -33,6 +34,9 @@ export class FilesSearchManager {
async search(user: UserModel, search: SearchFilesDto): Promise<FileContent[]> {
const [spaceIds, shareIds] = await Promise.all([this.spacesQueries.spaceIds(user.id), this.sharesQueries.shareIds(user.id, +user.isAdmin)])
if (search.fullText) {
+ if (!configuration.applications.files.contentIndexing) {
+ throw new HttpException('Full-text search is disabled', HttpStatus.BAD_REQUEST)
+ }
return await this.searchFullText(user.id, spaceIds, shareIds, search.content, search.limit)
} else {
return await this.searchFileNames(user.id, spaceIds, shareIds, search.content, search.limit)
diff --git a/environment/environment.dist.yaml b/environment/environment.dist.yaml
index 6f510ee2..dc21ce6f 100755
--- a/environment/environment.dist.yaml
+++ b/environment/environment.dist.yaml
@@ -138,6 +138,9 @@ applications:
dataPath: /home/sync-in
# default: 5368709120 (5 GB)
maxUploadSize: 5368709120
+ # Enable indexing of file contents for search (disabling this turns off full-text search)
+ # default: true
+ contentIndexing: true
# Show files starting with a dot in the file explorer
# default: false
showHiddenFiles: false
diff --git a/frontend/src/app/applications/search/components/search.component.html b/frontend/src/app/applications/search/components/search.component.html
index 8c801c57..f4deebd9 100644
--- a/frontend/src/app/applications/search/components/search.component.html
+++ b/frontend/src/app/applications/search/components/search.component.html
@@ -42,7 +42,7 @@
</div>
<div appAutoResize class="fs-xs">
@if (errorMessage) {
- <div class="p-3 text-danger fw-500">{{ errorMessage }}</div>
+ <div class="p-3 text-danger fw-500" l10nTranslate>{{ errorMessage }}</div>
}
@for (f of this.store.filesSearch() | searchFilter:inputFilter.search(); track $index) {
<div class="app-div-striped py-2" (click)="selectedId = $index" (appTap)="goTo(f)" [class.selected]="$index === selectedId">
diff --git a/frontend/src/app/applications/search/components/search.component.ts b/frontend/src/app/applications/search/components/search.component.ts
index 00b4d7b0..394deffc 100644
--- a/frontend/src/app/applications/search/components/search.component.ts
+++ b/frontend/src/app/applications/search/components/search.component.ts
@@ -12,7 +12,7 @@ import { FaIconComponent } from '@fortawesome/angular-fontawesome'
import { faFont, faSpinner, faTimes, faTrashCan } from '@fortawesome/free-solid-svg-icons'
import { minCharsToSearch } from '@sync-in-server/backend/src/applications/files/constants/indexing'
import type { SearchFilesDto } from '@sync-in-server/backend/src/applications/files/dto/file-operations.dto'
-import { L10N_LOCALE, L10nLocale, L10nTranslatePipe } from 'angular-l10n'
+import { L10N_LOCALE, L10nLocale, L10nTranslateDirective, L10nTranslatePipe } from 'angular-l10n'
import { ButtonCheckboxDirective } from 'ngx-bootstrap/buttons'
import { TooltipDirective } from 'ngx-bootstrap/tooltip'
import { FilterComponent } from '../../../common/components/filter.component'
@@ -39,7 +39,8 @@ import { SEARCH_ICON, SEARCH_PATH } from '../search.constants'
AutoResizeDirective,
TooltipDirective,
SearchFilterPipe,
- TapDirective
+ TapDirective,
+ L10nTranslateDirective
],
templateUrl: './search.component.html'
})
diff --git a/frontend/src/i18n/fr.json b/frontend/src/i18n/fr.json
index fe738702..f5c9fe36 100644
--- a/frontend/src/i18n/fr.json
+++ b/frontend/src/i18n/fr.json
@@ -29,6 +29,7 @@
"Search for content": "Rechercher du contenu",
"Search for files": "Rechercher des fichiers",
"Full-text search": "Recherche full-text",
+ "Full-text search is disabled": "La recherche full-text est désactivée",
"Type to search for users or groups to add": "Tapez pour rechercher des utilisateurs ou des groupes à ajouter",
"Type to search for groups to add": "Tapez pour rechercher des groupes à ajouter",
"Type to search for users to add": "Tapez pour rechercher des utilisateurs à ajouter",