Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
refactor(backend:files): content indexing & quota
johaven8180f0ffe0e7
diff --git a/backend/src/applications/files/adapters/files-indexer-mysql.service.spec.ts b/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.tssimilarity index 95%rename from backend/src/applications/files/adapters/files-indexer-mysql.service.spec.tsrename to backend/src/applications/files/adapters/files-content-store-mysql.service.spec.tsindex 22954976..b46e2f50 100644--- a/backend/src/applications/files/adapters/files-indexer-mysql.service.spec.ts+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts@@ -1,11 +1,11 @@import { Test, TestingModule } from '@nestjs/testing'import { Cache } from '../../../infrastructure/cache/services/cache.service'import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'-import { FilesIndexerMySQL } from './files-indexer-mysql.service'+import { FilesContentStoreMySQL } from './files-content-store-mysql.service'-describe(FilesIndexerMySQL.name, () => {+describe(FilesContentStoreMySQL.name, () => {let module: TestingModule- let filesIndexerMySQL: FilesIndexerMySQL+ let filesIndexerMySQL: FilesContentStoreMySQLlet db: { execute: jest.Mock }let cache: { genSlugKey: jest.Mock; get: jest.Mock; set: jest.Mock }@@ -18,11 +18,11 @@ describe(FilesIndexerMySQL.name, () => {}module = await Test.createTestingModule({- providers: [FilesIndexerMySQL, { provide: DB_TOKEN_PROVIDER, useValue: db }, { provide: Cache, useValue: cache }]+ providers: [FilesContentStoreMySQL, { provide: DB_TOKEN_PROVIDER, useValue: db }, { provide: Cache, useValue: cache }]}).compile()module.useLogger(['fatal'])- filesIndexerMySQL = module.get<FilesIndexerMySQL>(FilesIndexerMySQL)+ filesIndexerMySQL = module.get<FilesContentStoreMySQL>(FilesContentStoreMySQL)})afterAll(async () => {diff --git a/backend/src/applications/files/adapters/files-indexer-mysql.service.ts b/backend/src/applications/files/adapters/files-content-store-mysql.service.tssimilarity index 96%rename from backend/src/applications/files/adapters/files-indexer-mysql.service.tsrename to backend/src/applications/files/adapters/files-content-store-mysql.service.tsindex ae4ff85b..d0704372 100644--- a/backend/src/applications/files/adapters/files-indexer-mysql.service.ts+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.ts@@ -4,14 +4,14 @@ import { MySqlQueryResult } from 'drizzle-orm/mysql2'import { CacheDecorator } from '../../../infrastructure/cache/cache.decorator'import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'import { DBSchema } from '../../../infrastructure/database/interfaces/database.interface'-import { FilesIndexer } from '../models/files-indexer'+import { FilesContentStore } from '../models/files-content-store'import { FileContent } from '../schemas/file-content.interface'import { createTableFilesContent, FILES_CONTENT_TABLE_PREFIX } from '../schemas/files-content.schema'import { analyzeTerms, genTermsPattern, MaxSortedList } from '../utils/files-search'@Injectable()-export class FilesIndexerMySQL implements FilesIndexer {- private readonly logger = new Logger(FilesIndexerMySQL.name)+export class FilesContentStoreMySQL implements FilesContentStore {+ private readonly logger = new Logger(FilesContentStoreMySQL.name)constructor(@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema) {}diff --git a/backend/src/applications/files/constants/compress.ts b/backend/src/applications/files/constants/compress.tsindex 48e2beea..c2cc0814 100644--- a/backend/src/applications/files/constants/compress.ts+++ b/backend/src/applications/files/constants/compress.ts@@ -1,2 +1,2 @@-export const tarExtension = 'tar'-export const tarGzExtension = 'tgz'+export const TAR_EXTENSION = 'tar'+export const TAR_GZ_EXTENSION = 'tgz'diff --git a/backend/src/applications/files/constants/indexing.ts b/backend/src/applications/files/constants/indexing.tsindex fdf8cc99..03486c0b 100644--- a/backend/src/applications/files/constants/indexing.ts+++ b/backend/src/applications/files/constants/indexing.ts@@ -1,2 +1,4 @@-export const minCharsToSearch = 3-export const indexableExtensions = new Set(['docx', 'pptx', 'xlsx', 'odt', 'odp', 'ods', 'pdf', 'txt', 'md', 'html', 'htm'])+export const MIN_CHARS_TO_SEARCH = 3+export const INDEXABLE_EXTENSIONS = new Set(['docx', 'pptx', 'xlsx', 'odt', 'odp', 'ods', 'pdf', 'txt', 'md', 'html', 'htm'])+export const CACHE_INDEXING_UPDATE_PREFIX = 'event-update-indexing' as const+export const CACHE_INDEXING_TTL = 86400 // 1 daydiff --git a/backend/src/applications/files/dto/file-operations.dto.ts b/backend/src/applications/files/dto/file-operations.dto.tsindex 50f986bd..91ba66fb 100644--- a/backend/src/applications/files/dto/file-operations.dto.ts+++ b/backend/src/applications/files/dto/file-operations.dto.ts@@ -2,7 +2,7 @@ import { Transform } from 'class-transformer'import { ArrayMinSize, IsArray, IsBoolean, IsDefined, IsIn, IsInt, IsNotEmpty, IsOptional, IsString, IsUrl } from 'class-validator'import { RejectIfMatch } from '../../../common/decorators'import { regExpInvalidFileName } from '../../../common/shared'-import { tarExtension, tarGzExtension } from '../constants/compress'+import { TAR_EXTENSION, TAR_GZ_EXTENSION } from '../constants/compress'export class CopyMoveFileDto {@IsNotEmpty()@@ -58,8 +58,8 @@ export class CompressFileDto {@IsNotEmpty()@IsString()- @IsIn([tarExtension, tarGzExtension])- extension: typeof tarExtension | typeof tarGzExtension+ @IsIn([TAR_EXTENSION, TAR_GZ_EXTENSION])+ extension: typeof TAR_EXTENSION | typeof TAR_GZ_EXTENSION}export class SearchFilesDto {diff --git a/backend/src/applications/files/events/files-events.utils.ts b/backend/src/applications/files/events/files-events.utils.tsnew file mode 100644index 00000000..9d5b6678--- /dev/null+++ b/backend/src/applications/files/events/files-events.utils.ts@@ -0,0 +1,24 @@+import { FILE_REPOSITORY } from '../constants/operations'+import type { SpaceEnv } from '../../spaces/models/space-env.model'++export function SpaceToFileRepository(userId: number, space: SpaceEnv): { id: number; type: FILE_REPOSITORY } | null {+ if (space.inPersonalSpace) {+ // Personal user space+ return { id: userId, type: FILE_REPOSITORY.USER }+ } else if (space.root?.externalPath) {+ // External paths used as shares or as space roots share the same quota as their origin+ if (space.inSharesRepository) {+ return { id: space.root?.externalParentShareId || space.id, type: FILE_REPOSITORY.SHARE }+ }+ return { id: space.id, type: FILE_REPOSITORY.SPACE }+ } else if (space.root?.file?.path && space.root.owner?.login) {+ // Space root is linked to a user file+ return { id: space.root.owner.id, type: FILE_REPOSITORY.USER }+ } else if (space.root?.file?.space?.id) {+ return { id: space.root.file.space.id, type: FILE_REPOSITORY.SPACE }+ } else if (space.id) {+ return { id: space.id, type: FILE_REPOSITORY.SPACE }+ } else {+ return null+ }+}diff --git a/backend/src/applications/files/files.module.ts b/backend/src/applications/files/files.module.tsindex 111875d4..6c8638a5 100644--- a/backend/src/applications/files/files.module.ts+++ b/backend/src/applications/files/files.module.ts@@ -1,12 +1,12 @@import { Module } from '@nestjs/common'import { configuration } from '../../configuration/config.environment'-import { FilesIndexerMySQL } from './adapters/files-indexer-mysql.service'+import { FilesContentStoreMySQL } from './adapters/files-content-store-mysql.service'import { FilesTasksController } from './files-tasks.controller'import { FilesController } from './files.controller'-import { FilesIndexer } from './models/files-indexer'+import { FilesContentStore } from './models/files-content-store'import { CollaboraOnlineModule } from './modules/collabora-online/collabora-online.module'import { OnlyOfficeModule } from './modules/only-office/only-office.module'-import { FilesContentManager } from './services/files-content-manager.service'+import { FilesContentIndexer } from './services/files-content-indexer.service'import { FilesLockManager } from './services/files-lock-manager.service'import { FilesManager } from './services/files-manager.service'import { FilesMethods } from './services/files-methods.service'@@ -34,8 +34,8 @@ import { FilesQuotaManager } from './services/files-quota-manager.service'FilesScheduler,FilesRecents,FilesParser,- FilesContentManager,- { provide: FilesIndexer, useClass: FilesIndexerMySQL },+ FilesContentIndexer,+ { provide: FilesContentStore, useClass: FilesContentStoreMySQL },FilesSearchManager,FilesEventManager,FilesQuotaManagerdiff --git a/backend/src/applications/files/interfaces/file-event.interface.ts b/backend/src/applications/files/interfaces/file-event.interface.tsindex b5e5cc0a..42bb61dc 100644--- a/backend/src/applications/files/interfaces/file-event.interface.ts+++ b/backend/src/applications/files/interfaces/file-event.interface.ts@@ -12,7 +12,6 @@ export interface FileEventType {space: SpaceEnvaction: ACTIONrPath: string- oriPath?: string // src from copy/move operations}export interface FileEventEmit {diff --git a/backend/src/applications/files/models/files-indexer.ts b/backend/src/applications/files/models/files-content-store.tssimilarity index 95%rename from backend/src/applications/files/models/files-indexer.tsrename to backend/src/applications/files/models/files-content-store.tsindex 68bbb78c..437bc7ca 100644--- a/backend/src/applications/files/models/files-indexer.ts+++ b/backend/src/applications/files/models/files-content-store.ts@@ -1,6 +1,6 @@import { FileContent } from '../schemas/file-content.interface'-export abstract class FilesIndexer {+export abstract class FilesContentStore {abstract indexesList(): Promise<string[]>abstract getIndexName(indexSuffix: string): stringdiff --git a/backend/src/applications/files/services/files-content-manager.service.ts b/backend/src/applications/files/services/files-content-indexer.service.tssimilarity index 86%rename from backend/src/applications/files/services/files-content-manager.service.tsrename to backend/src/applications/files/services/files-content-indexer.service.tsindex c609aacd..4012e5e3 100644--- a/backend/src/applications/files/services/files-content-manager.service.ts+++ b/backend/src/applications/files/services/files-content-indexer.service.ts@@ -2,28 +2,28 @@ import { Injectable, Logger } from '@nestjs/common'import fs from 'fs/promises'import { Stats } from 'node:fs'import path from 'node:path'-import { indexableExtensions } from '../constants/indexing'+import { INDEXABLE_EXTENSIONS } from '../constants/indexing'import { FileIndexContext, FileParseContext } from '../interfaces/file-parse-index'-import { FilesIndexer } from '../models/files-indexer'+import { FilesContentStore } from '../models/files-content-store'import { FileContent } from '../schemas/file-content.interface'import { docTextify } from '../utils/doc-textify/doc-textify'import { PdfOCRWorkerManager } from '../utils/doc-textify/utils/pdf-ocr'-import { getMimeType } from '../utils/files'+import { getExtensionWithoutDot, getMimeType } from '../utils/files'import { FilesParser } from './files-parser.service'-import { FILE_REPOSITORY } from '../constants/operations'+import { genIndexingKey } from '../utils/indexing'@Injectable()-export class FilesContentManager {+export class FilesContentIndexer {private readonly maxDocumentSize = 150 * 1_000_000- private readonly logger = new Logger(FilesContentManager.name)+ private readonly logger = new Logger(FilesContentIndexer.name)private pdfOcrWorkerManager: PdfOCRWorkerManager | null = nullconstructor(- private readonly filesIndexer: FilesIndexer,+ private readonly filesIndexer: FilesContentStore,private readonly filesParser: FilesParser) {}- async parseAndIndexAllFiles(): Promise<void> {+ async parseAndIndexAllFiles(userIds?: number[], spaceIds?: number[], shareIds?: number[]): Promise<void> {this.pdfOcrWorkerManager = PdfOCRWorkerManager.getInstance(this.logger)try {await this.pdfOcrWorkerManager.start()@@ -32,18 +32,8 @@ export class FilesContentManager {}try {const indexSuffixes: string[] = []- for await (const [id, type, paths] of this.filesParser.allPaths()) {- let indexSuffix: string- switch (type) {- case FILE_REPOSITORY.USER:- indexSuffix = `${FILE_REPOSITORY.USER}_${id}`- break- case FILE_REPOSITORY.SPACE:- indexSuffix = `${FILE_REPOSITORY.SPACE}_${id}`- break- case FILE_REPOSITORY.SHARE:- indexSuffix = `${FILE_REPOSITORY.SHARE}_${id}`- }+ for await (const [id, type, paths] of this.filesParser.allPaths(userIds, spaceIds, shareIds)) {+ const indexSuffix = genIndexingKey(id, type)try {await this.indexFiles(indexSuffix, paths)} catch (e) {@@ -51,8 +41,10 @@ export class FilesContentManager {}indexSuffixes.push(indexSuffix)}- // clean up old tables- await this.filesIndexer.cleanIndexes(indexSuffixes)+ // clean up old tables only when all indexes have been indexed+ if (!userIds?.length && !spaceIds?.length && !shareIds?.length) {+ await this.filesIndexer.cleanIndexes(indexSuffixes)+ }} finally {await this.pdfOcrWorkerManager?.stop()this.pdfOcrWorkerManager = null@@ -142,8 +134,8 @@ export class FilesContentManager {}private async analyzeFile(realPath: string, context: FileIndexContext, isRootFile = false): Promise<FileContent> {- const extension = path.extname(realPath).slice(1).toLowerCase()- if (!indexableExtensions.has(extension)) return null+ const extension = getExtensionWithoutDot(realPath)+ if (!INDEXABLE_EXTENSIONS.has(extension)) return nullconst fileName = isRootFile ? path.basename(context.pathPrefix) : path.basename(realPath)diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.tsindex f85c8f67..dc56c63e 100644--- a/backend/src/applications/files/services/files-manager.service.ts+++ b/backend/src/applications/files/services/files-manager.service.ts@@ -23,7 +23,7 @@ import { canAccessToSpace, haveSpaceEnvPermissions } from '../../spaces/utils/peimport { UserModel } from '../../users/models/user.model'import { DEPTH, LOCK_DEPTH } from '../../webdav/constants/webdav'import { CACHE_LOCK_FILE_TTL } from '../constants/cache'-import { tarGzExtension } from '../constants/compress'+import { TAR_GZ_EXTENSION } from '../constants/compress'import { COMPRESSION_EXTENSION, DEFAULT_HIGH_WATER_MARK } from '../constants/files'import { FILE_OPERATION } from '../constants/operations'import { DOCUMENT_TYPE, SAMPLE_PATH_WITHOUT_EXT } from '../constants/samples'@@ -566,7 +566,7 @@ export class FilesManager {const archiveExt = dto.name.endsWith(dto.extension) ? '' : `.${dto.extension}`const dstPath = await uniqueFilePathFromDir(path.join(dto.compressInDirectory ? srcPath : user.tasksPath, `${dto.name}${archiveExt}`))const archive: Archiver = archiver('tar', {- gzip: dto.extension === tarGzExtension,+ gzip: dto.extension === TAR_GZ_EXTENSION,gzipOptions: {level: 9}@@ -641,7 +641,7 @@ export class FilesManager {await extractTar({file: space.realPath,cwd: dstPath,- gzip: COMPRESSION_EXTENSION.get(extension) === tarGzExtension,+ gzip: COMPRESSION_EXTENSION.get(extension) === TAR_GZ_EXTENSION,preserveOwner: false})}diff --git a/backend/src/applications/files/services/files-methods.service.spec.ts b/backend/src/applications/files/services/files-methods.service.spec.tsindex 3aae7a7d..33fdd35c 100644--- a/backend/src/applications/files/services/files-methods.service.spec.ts+++ b/backend/src/applications/files/services/files-methods.service.spec.ts@@ -14,7 +14,7 @@ import { SpacesQueries } from '../../spaces/services/spaces-queries.service'import { UserModel } from '../../users/models/user.model'import { UsersQueries } from '../../users/services/users-queries.service'import { generateUserTest } from '../../users/utils/test'-import { tarExtension } from '../constants/compress'+import { TAR_EXTENSION } from '../constants/compress'import { CompressFileDto, CopyMoveFileDto } from '../dto/file-operations.dto'import { FilesManager } from './files-manager.service'import { FilesMethods } from './files-methods.service'@@ -97,7 +97,7 @@ describe(FilesMethods.name, () => {name: '../../archive',compressInDirectory: false,files: [{ name: '../../foo', rootAlias: undefined }],- extension: tarExtension+ extension: TAR_EXTENSION}expect(() => transformAndValidate(CompressFileDto, compressFileDto)).toThrow()await expect(filesMethods.compress(userTest, spaceEnv, compressFileDto)).rejects.toThrow(/does not exist/i)diff --git a/backend/src/applications/files/services/files-parser.service.ts b/backend/src/applications/files/services/files-parser.service.tsindex aee1ad26..2364f08e 100644--- a/backend/src/applications/files/services/files-parser.service.ts+++ b/backend/src/applications/files/services/files-parser.service.ts@@ -24,20 +24,20 @@ export class FilesParser {constructor(@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema) {}- async *allPaths(userId?: number, spaceIds?: number[], shareIds?: number[]): AsyncGenerator<[number, FILE_REPOSITORY, FileParseContext[]]> {- yield* this.userPaths(userId)+ async *allPaths(userIds?: number[], spaceIds?: number[], shareIds?: number[]): AsyncGenerator<[number, FILE_REPOSITORY, FileParseContext[]]> {+ yield* this.userPaths(userIds)yield* this.spacePaths(spaceIds)yield* this.sharePaths(shareIds)}- async *userPaths(userId?: number): AsyncGenerator<[number, FILE_REPOSITORY, FileParseContext[]]> {+ async *userPaths(userIds?: number[]): AsyncGenerator<[number, FILE_REPOSITORY, FileParseContext[]]> {for (const user of await this.db.select({id: users.id,login: users.login}).from(users)- .where(and(...[eq(users.storageIndexing, true), lte(users.role, USER_ROLE.USER), ...(userId ? [eq(users.id, userId)] : [])]))) {+ .where(and(...[eq(users.storageIndexing, true), lte(users.role, USER_ROLE.USER), ...(userIds ? [inArray(users.id, userIds)] : [])]))) {const userFilesPath = UserModel.getFilesPath(user.login)if (!(await isPathExists(userFilesPath))) {this.logger.warn({ tag: this.userPaths.name, msg: `user path does not exist : ${userFilesPath}` })diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.tsindex 2777d449..2bf3cbfc 100644--- a/backend/src/applications/files/services/files-scheduler.service.ts+++ b/backend/src/applications/files/services/files-scheduler.service.ts@@ -17,7 +17,7 @@ import { FileTask, FileTaskStatus } from '../models/file-task'import { filesRecents } from '../schemas/files-recents.schema'import { files } from '../schemas/files.schema'import { dirHasChildren, isPathExists, removeFiles } from '../utils/files'-import { FilesContentManager } from './files-content-manager.service'+import { FilesContentIndexer } from './files-content-indexer.service'import { FilesTasksManager } from './files-tasks-manager.service'import { FilesQuotaManager } from './files-quota-manager.service'@@ -31,7 +31,7 @@ export class FilesScheduler {constructor(@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema,private readonly cache: Cache,- private readonly filesContentManager: FilesContentManager,+ private readonly filesContentIndexer: FilesContentIndexer,private readonly filesQuotaManager: FilesQuotaManager) {}@@ -46,7 +46,7 @@ export class FilesScheduler {}}- @Timeout(180_000)+ @Timeout(300_000)async afterStartup(): Promise<void> {try {await this.indexContentFiles()@@ -161,7 +161,7 @@ export class FilesScheduler {this.isIndexContentFilesRunning = truethis.logger.log({ tag: this.indexContentFiles.name, msg: `START` })try {- await this.filesContentManager.parseAndIndexAllFiles()+ await this.filesContentIndexer.parseAndIndexAllFiles()this.logger.log({ tag: this.indexContentFiles.name, msg: `END` })} finally {this.isIndexContentFilesRunning = falsediff --git a/backend/src/applications/files/services/files-search-manager.service.spec.ts b/backend/src/applications/files/services/files-search-manager.service.spec.tsindex 95b99383..cf9f416a 100644--- a/backend/src/applications/files/services/files-search-manager.service.spec.ts+++ b/backend/src/applications/files/services/files-search-manager.service.spec.ts@@ -1,7 +1,7 @@import { Test, TestingModule } from '@nestjs/testing'import { SharesQueries } from '../../shares/services/shares-queries.service'import { SpacesQueries } from '../../spaces/services/spaces-queries.service'-import { FilesIndexer } from '../models/files-indexer'+import { FilesContentStore } from '../models/files-content-store'import { FilesParser } from './files-parser.service'import { FilesSearchManager } from './files-search-manager.service'@@ -12,7 +12,7 @@ describe(FilesSearchManager.name, () => {const module: TestingModule = await Test.createTestingModule({providers: [FilesSearchManager,- { provide: FilesIndexer, useValue: {} },+ { provide: FilesContentStore, useValue: {} },{ provide: FilesParser, useValue: {} },{provide: SpacesQueries,diff --git a/backend/src/applications/files/services/files-search-manager.service.ts b/backend/src/applications/files/services/files-search-manager.service.tsindex 765029d5..b6f65a23 100644--- a/backend/src/applications/files/services/files-search-manager.service.ts+++ b/backend/src/applications/files/services/files-search-manager.service.ts@@ -7,19 +7,20 @@ import { SharesQueries } from '../../shares/services/shares-queries.service'import { SpacesQueries } from '../../spaces/services/spaces-queries.service'import { UserModel } from '../../users/models/user.model'import { SearchFilesDto } from '../dto/file-operations.dto'-import { FilesIndexer } from '../models/files-indexer'+import { FilesContentStore } from '../models/files-content-store'import { FileContent } from '../schemas/file-content.interface'import { dirName, fileName, getMimeType } from '../utils/files'import { genRegexPositiveAndNegativeTerms } from '../utils/files-search'import { FilesParser } from './files-parser.service'import { FILE_REPOSITORY } from '../constants/operations'+import { genIndexingKey } from '../utils/indexing'@Injectable()export class FilesSearchManager {private readonly logger = new Logger(FilesSearchManager.name)constructor(- private readonly filesIndexer: FilesIndexer,+ private readonly filesIndexer: FilesContentStore,private readonly filesParser: FilesParser,private readonly spacesQueries: SpacesQueries,private readonly sharesQueries: SharesQueries@@ -39,9 +40,9 @@ export class FilesSearchManager {private async searchFullText(userId: number, spaceIds: number[], shareIds: number[], search: string, limit: number): Promise<FileContent[]> {const indexNames = await this.filesIndexer.existingIndexes([- `${FILE_REPOSITORY.USER}_${userId}`,- ...spaceIds.map((id) => `${FILE_REPOSITORY.SPACE}_${id}`),- ...shareIds.map((id) => `${FILE_REPOSITORY.SHARE}_${id}`)+ genIndexingKey(userId, FILE_REPOSITORY.USER),+ ...spaceIds.map((id) => genIndexingKey(id, FILE_REPOSITORY.SPACE)),+ ...shareIds.map((id) => genIndexingKey(id, FILE_REPOSITORY.SHARE))])if (indexNames.length === 0) {return []@@ -63,7 +64,7 @@ export class FilesSearchManager {private async searchFileNames(userId: number, spaceIds: number[], shareIds: number[], search: string, limit: number): Promise<FileContent[]> {const fileContents: FileContent[] = []const regexpTerms = genRegexPositiveAndNegativeTerms(search)- for await (const [_id, _type, paths] of this.filesParser.allPaths(userId, spaceIds, shareIds)) {+ for await (const [_id, _type, paths] of this.filesParser.allPaths([userId], spaceIds, shareIds)) {for (const p of paths) {const regexBasePath = new RegExp(`^/?${p.realPath}/?`)if (!p.isDir) {diff --git a/backend/src/applications/files/utils/doc-textify/doc-textify.ts b/backend/src/applications/files/utils/doc-textify/doc-textify.tsindex df65c5eb..27bd8fa9 100644--- a/backend/src/applications/files/utils/doc-textify/doc-textify.ts+++ b/backend/src/applications/files/utils/doc-textify/doc-textify.ts@@ -1,5 +1,4 @@import fs from 'node:fs/promises'-import path from 'node:path'import { parseExcel } from './adapters/excel'import { parseHtml } from './adapters/html'import { parseMarkdown } from './adapters/markdown'@@ -10,6 +9,8 @@ import { parseText } from './adapters/text'import { parseWord } from './adapters/word'import { DocTextifyOptions } from './interfaces/doc-textify.interfaces'import { cleanContent } from './utils/clean'+import { INDEXABLE_EXTENSIONS } from '../../constants/indexing'+import { getExtensionWithoutDot } from '../files'/** Main: determine parser by extension and dispatch */export async function docTextify(@@ -31,7 +32,11 @@ export async function docTextify(}}- const ext = fileProperties?.extension || path.extname(filePath).slice(1).toLowerCase()+ const ext = fileProperties?.extension || getExtensionWithoutDot(filePath)++ if (!INDEXABLE_EXTENSIONS.has(ext)) {+ throw new Error(`currently only supports ${[...INDEXABLE_EXTENSIONS].join(',')} files`)+ }switch (ext) {case 'docx':@@ -54,6 +59,6 @@ export async function docTextify(case 'htm':return cleanContent(await parseHtml(filePath), options)default:- throw new Error('currently only supports docx, pptx, xlsx, odt, odp, ods, pdf, txt, md, html files')+ throw new Error(`no handler found for extension: ${ext}`)}}diff --git a/backend/src/applications/files/utils/files-search.ts b/backend/src/applications/files/utils/files-search.tsindex b9dab0ee..d42d3d8c 100644--- a/backend/src/applications/files/utils/files-search.ts+++ b/backend/src/applications/files/utils/files-search.ts@@ -1,5 +1,5 @@import { escapeString } from '../../../common/functions'-import { minCharsToSearch } from '../constants/indexing'+import { MIN_CHARS_TO_SEARCH } from '../constants/indexing'const regexMatchSearchBoolean = new RegExp(`([+-]?)(?:"([^"]+)"|(\\S+))`)const regexMatchesSearchBoolean = new RegExp(regexMatchSearchBoolean.source, 'g')@@ -59,7 +59,7 @@ export function analyzeTerms(search: string, onlyAllowNegative = false): string[const [, operator, quoted, unquoted] = match.match(regexMatchSearchBoolean)let term: string = (quoted || unquoted).trim()- if (term.length < minCharsToSearch) return null+ if (term.length < MIN_CHARS_TO_SEARCH) return nullif ((onlyAllowNegative && operator !== '-') || (!onlyAllowNegative && (operator === '-' || operator === '~'))) return nulldiff --git a/backend/src/applications/files/utils/files.ts b/backend/src/applications/files/utils/files.tsindex a143a3ef..841603e7 100644--- a/backend/src/applications/files/utils/files.ts+++ b/backend/src/applications/files/utils/files.ts@@ -97,6 +97,10 @@ export function getMimeType(fPath: string, isDir: boolean): string {return 'file'}+export function getExtensionWithoutDot(fPath: string): string {+ return path.extname(fPath).slice(1).toLowerCase()+}+export function genEtag(file?: Pick<FileProps, 'size' | 'mtime'>, rPath?: string, weakPrefix = true): string {if (!file) {if (!rPath) throw new Error('File or path are missing')diff --git a/backend/src/applications/files/utils/indexing.ts b/backend/src/applications/files/utils/indexing.tsnew file mode 100644index 00000000..c2190f8e--- /dev/null+++ b/backend/src/applications/files/utils/indexing.ts@@ -0,0 +1,15 @@+import { FILE_REPOSITORY } from '../constants/operations'+import { CACHE_INDEXING_UPDATE_PREFIX } from '../constants/indexing'+import { SpaceEnv } from '../../spaces/models/space-env.model'+import { SpaceToFileRepository } from '../events/files-events.utils'++export function genIndexingKey(id: number, type: FILE_REPOSITORY, sep = '_'): string {+ return `${type}${sep}${id}`+}++export function indexingUpdateCacheKeyFromSpace(userId: number, space: SpaceEnv): string | null {+ const repository: { id: number; type: FILE_REPOSITORY } = SpaceToFileRepository(userId, space)+ if (repository === null) return null+ const indexingKey = genIndexingKey(repository.id, repository.type, '-')+ return `${CACHE_INDEXING_UPDATE_PREFIX}-${indexingKey}`+}diff --git a/backend/src/applications/files/utils/quota.ts b/backend/src/applications/files/utils/quota.tsindex 3f8093e9..bf3b76a9 100644--- a/backend/src/applications/files/utils/quota.ts+++ b/backend/src/applications/files/utils/quota.ts@@ -1,29 +1,13 @@import { SpaceEnv } from '../../spaces/models/space-env.model'import { CACHE_QUOTA_EVENT_UPDATE_PREFIX, CACHE_QUOTA_PREFIX } from '../constants/cache'import { FILE_REPOSITORY } from '../constants/operations'+import { SpaceToFileRepository } from '../events/files-events.utils'export function genQuotaCacheKey(id: number, type: FILE_REPOSITORY, isEventUpdate = false): string {return `${isEventUpdate ? CACHE_QUOTA_EVENT_UPDATE_PREFIX : CACHE_QUOTA_PREFIX}-${type}-${id}`}-export function quotaCacheKeyFromSpace(userId: number, space: SpaceEnv, isEventUpdate = false): string {- if (space.inPersonalSpace) {- // Personal user space- return genQuotaCacheKey(userId, FILE_REPOSITORY.USER, isEventUpdate)- } else if (space.root?.externalPath) {- // External paths used as shares or as space roots share the same quota as their origin- if (space.inSharesRepository) {- return genQuotaCacheKey(space.root?.externalParentShareId || space.id, FILE_REPOSITORY.SHARE, isEventUpdate)- }- return genQuotaCacheKey(space.id, FILE_REPOSITORY.SPACE, isEventUpdate)- } else if (space.root?.file?.path && space.root.owner?.login) {- // Space root is linked to a user file- return genQuotaCacheKey(space.root.owner.id, FILE_REPOSITORY.USER, isEventUpdate)- } else if (space.root?.file?.space?.id) {- return genQuotaCacheKey(space.root.file.space.id, FILE_REPOSITORY.SPACE, isEventUpdate)- } else if (space.id) {- return genQuotaCacheKey(space.id, FILE_REPOSITORY.SPACE, isEventUpdate)- } else {- return null- }+export function quotaCacheKeyFromSpace(userId: number, space: SpaceEnv, isEventUpdate = false): string | null {+ const repository: { id: number; type: FILE_REPOSITORY } = SpaceToFileRepository(userId, space)+ return repository === null ? null : genQuotaCacheKey(repository.id, repository.type, isEventUpdate)}diff --git a/frontend/src/app/applications/files/components/dialogs/files-compression-dialog.component.ts b/frontend/src/app/applications/files/components/dialogs/files-compression-dialog.component.tsindex c3df4244..0b15df6b 100644--- a/frontend/src/app/applications/files/components/dialogs/files-compression-dialog.component.ts+++ b/frontend/src/app/applications/files/components/dialogs/files-compression-dialog.component.ts@@ -2,7 +2,7 @@ import { ChangeDetectionStrategy, Component, EventEmitter, HostListener, inject,import { FormsModule } from '@angular/forms'import { FaIconComponent } from '@fortawesome/angular-fontawesome'import { faFileArchive } from '@fortawesome/free-solid-svg-icons'-import { tarExtension, tarGzExtension } from '@sync-in-server/backend/src/applications/files/constants/compress'+import { TAR_EXTENSION, TAR_GZ_EXTENSION } from '@sync-in-server/backend/src/applications/files/constants/compress'import type { CompressFileDto } from '@sync-in-server/backend/src/applications/files/dto/file-operations.dto'import { L10N_LOCALE, L10nLocale, L10nTranslateDirective, L10nTranslatePipe } from 'angular-l10n'import { AutofocusDirective } from '../../../../common/directives/auto-focus.directive'@@ -16,7 +16,7 @@ import { FilesService } from '../../services/files.service'changeDetection: ChangeDetectionStrategy.OnPush})export class FilesCompressionDialogComponent implements OnInit {- @Input() archiveProps: CompressFileDto = { name: '', files: [], compressInDirectory: true, extension: tarExtension }+ @Input() archiveProps: CompressFileDto = { name: '', files: [], compressInDirectory: true, extension: TAR_EXTENSION }@Output() submitEvent = new EventEmitter()public disableInDirCompression = falseprotected readonly locale = inject<L10nLocale>(L10N_LOCALE)@@ -47,6 +47,6 @@ export class FilesCompressionDialogComponent implements OnInit {}setCompression(enable: boolean) {- this.archiveProps.extension = enable ? tarGzExtension : tarExtension+ this.archiveProps.extension = enable ? TAR_GZ_EXTENSION : TAR_EXTENSION}}diff --git a/frontend/src/app/applications/files/components/sidebar/files-clipboard.component.ts b/frontend/src/app/applications/files/components/sidebar/files-clipboard.component.tsindex c1e16867..f5fc7c18 100644--- a/frontend/src/app/applications/files/components/sidebar/files-clipboard.component.ts+++ b/frontend/src/app/applications/files/components/sidebar/files-clipboard.component.ts@@ -3,7 +3,7 @@ import { Component, inject, OnDestroy } from '@angular/core'import { FormsModule } from '@angular/forms'import { FaIconComponent } from '@fortawesome/angular-fontawesome'import { faArrowsAlt, faClone, faDownload, faQuestion, faTimes, faTrashCan } from '@fortawesome/free-solid-svg-icons'-import { tarExtension } from '@sync-in-server/backend/src/applications/files/constants/compress'+import { TAR_EXTENSION } from '@sync-in-server/backend/src/applications/files/constants/compress'import { FILE_OPERATION } from '@sync-in-server/backend/src/applications/files/constants/operations'import type { CompressFileDto } from '@sync-in-server/backend/src/applications/files/dto/file-operations.dto'import { L10N_LOCALE, L10nLocale, L10nTranslateDirective, L10nTranslatePipe } from 'angular-l10n'@@ -70,7 +70,7 @@ export class FilesClipboardComponent implements OnDestroy {name: this.files[0].name,compressInDirectory: this.operations[this.selectedAction].operation === FILE_OPERATION.COMPRESS,files: this.files.map((f: FileModel) => ({ name: f.name, rootAlias: f.root?.alias, path: f.path })),- extension: tarExtension+ extension: TAR_EXTENSION}const modalRef: BsModalRef<FilesCompressionDialogComponent> = this.layout.openDialog(FilesCompressionDialogComponent, null, {initialState: { archiveProps: archiveProps } as FilesCompressionDialogComponentdiff --git a/frontend/src/app/applications/search/components/search.component.ts b/frontend/src/app/applications/search/components/search.component.tsindex 204e7cd6..91a995c1 100644--- a/frontend/src/app/applications/search/components/search.component.ts+++ b/frontend/src/app/applications/search/components/search.component.ts@@ -4,7 +4,7 @@ import { FormsModule } from '@angular/forms'import { Router } from '@angular/router'import { FaIconComponent } from '@fortawesome/angular-fontawesome'import { faFont, faMapMarkerAlt, faSpinner, faTimes, faTrashCan } from '@fortawesome/free-solid-svg-icons'-import { minCharsToSearch } from '@sync-in-server/backend/src/applications/files/constants/indexing'+import { MIN_CHARS_TO_SEARCH } from '@sync-in-server/backend/src/applications/files/constants/indexing'import type { SearchFilesDto } from '@sync-in-server/backend/src/applications/files/dto/file-operations.dto'import { L10N_LOCALE, L10nLocale, L10nTranslateDirective, L10nTranslatePipe } from 'angular-l10n'import { ButtonCheckboxDirective } from 'ngx-bootstrap/buttons'@@ -45,7 +45,7 @@ export class SearchComponent {protected readonly store = inject(StoreService)public searchContent: Signal<string> = computed(() => this.store.currentSearch().content)protected readonly icons = { SEARCH_ICON, faSpinner, faTrashCan, faTimes, faFont, faMapMarkerAlt }- protected minCharsToSearch = minCharsToSearch+ protected minCharsToSearch = MIN_CHARS_TO_SEARCHprotected loading = falseprotected errorMessage: string = nullprotected selectedId: number = nulldiff --git a/frontend/src/app/applications/spaces/components/spaces-browser.component.ts b/frontend/src/app/applications/spaces/components/spaces-browser.component.tsindex e41304f9..3855e93a 100644--- a/frontend/src/app/applications/spaces/components/spaces-browser.component.ts+++ b/frontend/src/app/applications/spaces/components/spaces-browser.component.ts@@ -31,7 +31,7 @@ import {faUpload} from '@fortawesome/free-solid-svg-icons'import { ContextMenuComponent, ContextMenuModule } from '@perfectmemory/ngx-contextmenu'-import { tarExtension } from '@sync-in-server/backend/src/applications/files/constants/compress'+import { TAR_EXTENSION } from '@sync-in-server/backend/src/applications/files/constants/compress'import { FILE_OPERATION } from '@sync-in-server/backend/src/applications/files/constants/operations'import type { CompressFileDto } from '@sync-in-server/backend/src/applications/files/dto/file-operations.dto'import type { FileProps } from '@sync-in-server/backend/src/applications/files/interfaces/file-props.interface'@@ -609,7 +609,7 @@ export class SpacesBrowserComponent implements OnInit, AfterViewInit, OnDestroyname: this.selection[0].name,compressInDirectory: this.inSharesList ? false : inDir,files: this.selection.map((f: FileModel) => ({ name: f.name, rootAlias: f.root?.alias })),- extension: tarExtension+ extension: TAR_EXTENSION}this.layout.openDialog(FilesCompressionDialogComponent, null, {initialState: {