Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

refactor(backend:files): content indexing & quota

johaven8180f0ffe0e7
diff --git a/backend/src/applications/files/adapters/files-indexer-mysql.service.spec.ts b/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts
similarity index 95%
rename from backend/src/applications/files/adapters/files-indexer-mysql.service.spec.ts
rename to backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts
index 22954976..b46e2f50 100644
--- a/backend/src/applications/files/adapters/files-indexer-mysql.service.spec.ts
+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts
@@ -1,11 +1,11 @@
import { Test, TestingModule } from '@nestjs/testing'
import { Cache } from '../../../infrastructure/cache/services/cache.service'
import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'
-import { FilesIndexerMySQL } from './files-indexer-mysql.service'
+import { FilesContentStoreMySQL } from './files-content-store-mysql.service'
-describe(FilesIndexerMySQL.name, () => {
+describe(FilesContentStoreMySQL.name, () => {
let module: TestingModule
- let filesIndexerMySQL: FilesIndexerMySQL
+ let filesIndexerMySQL: FilesContentStoreMySQL
let db: { execute: jest.Mock }
let cache: { genSlugKey: jest.Mock; get: jest.Mock; set: jest.Mock }
@@ -18,11 +18,11 @@ describe(FilesIndexerMySQL.name, () => {
}
module = await Test.createTestingModule({
- providers: [FilesIndexerMySQL, { provide: DB_TOKEN_PROVIDER, useValue: db }, { provide: Cache, useValue: cache }]
+ providers: [FilesContentStoreMySQL, { provide: DB_TOKEN_PROVIDER, useValue: db }, { provide: Cache, useValue: cache }]
}).compile()
module.useLogger(['fatal'])
- filesIndexerMySQL = module.get<FilesIndexerMySQL>(FilesIndexerMySQL)
+ filesIndexerMySQL = module.get<FilesContentStoreMySQL>(FilesContentStoreMySQL)
})
afterAll(async () => {
diff --git a/backend/src/applications/files/adapters/files-indexer-mysql.service.ts b/backend/src/applications/files/adapters/files-content-store-mysql.service.ts
similarity index 96%
rename from backend/src/applications/files/adapters/files-indexer-mysql.service.ts
rename to backend/src/applications/files/adapters/files-content-store-mysql.service.ts
index ae4ff85b..d0704372 100644
--- a/backend/src/applications/files/adapters/files-indexer-mysql.service.ts
+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.ts
@@ -4,14 +4,14 @@ import { MySqlQueryResult } from 'drizzle-orm/mysql2'
import { CacheDecorator } from '../../../infrastructure/cache/cache.decorator'
import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'
import { DBSchema } from '../../../infrastructure/database/interfaces/database.interface'
-import { FilesIndexer } from '../models/files-indexer'
+import { FilesContentStore } from '../models/files-content-store'
import { FileContent } from '../schemas/file-content.interface'
import { createTableFilesContent, FILES_CONTENT_TABLE_PREFIX } from '../schemas/files-content.schema'
import { analyzeTerms, genTermsPattern, MaxSortedList } from '../utils/files-search'
@Injectable()
-export class FilesIndexerMySQL implements FilesIndexer {
- private readonly logger = new Logger(FilesIndexerMySQL.name)
+export class FilesContentStoreMySQL implements FilesContentStore {
+ private readonly logger = new Logger(FilesContentStoreMySQL.name)
constructor(@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema) {}
diff --git a/backend/src/applications/files/constants/compress.ts b/backend/src/applications/files/constants/compress.ts
index 48e2beea..c2cc0814 100644
--- a/backend/src/applications/files/constants/compress.ts
+++ b/backend/src/applications/files/constants/compress.ts
@@ -1,2 +1,2 @@
-export const tarExtension = 'tar'
-export const tarGzExtension = 'tgz'
+export const TAR_EXTENSION = 'tar'
+export const TAR_GZ_EXTENSION = 'tgz'
diff --git a/backend/src/applications/files/constants/indexing.ts b/backend/src/applications/files/constants/indexing.ts
index fdf8cc99..03486c0b 100644
--- a/backend/src/applications/files/constants/indexing.ts
+++ b/backend/src/applications/files/constants/indexing.ts
@@ -1,2 +1,4 @@
-export const minCharsToSearch = 3
-export const indexableExtensions = new Set(['docx', 'pptx', 'xlsx', 'odt', 'odp', 'ods', 'pdf', 'txt', 'md', 'html', 'htm'])
+export const MIN_CHARS_TO_SEARCH = 3
+export const INDEXABLE_EXTENSIONS = new Set(['docx', 'pptx', 'xlsx', 'odt', 'odp', 'ods', 'pdf', 'txt', 'md', 'html', 'htm'])
+export const CACHE_INDEXING_UPDATE_PREFIX = 'event-update-indexing' as const
+export const CACHE_INDEXING_TTL = 86400 // 1 day
diff --git a/backend/src/applications/files/dto/file-operations.dto.ts b/backend/src/applications/files/dto/file-operations.dto.ts
index 50f986bd..91ba66fb 100644
--- a/backend/src/applications/files/dto/file-operations.dto.ts
+++ b/backend/src/applications/files/dto/file-operations.dto.ts
@@ -2,7 +2,7 @@ import { Transform } from 'class-transformer'
import { ArrayMinSize, IsArray, IsBoolean, IsDefined, IsIn, IsInt, IsNotEmpty, IsOptional, IsString, IsUrl } from 'class-validator'
import { RejectIfMatch } from '../../../common/decorators'
import { regExpInvalidFileName } from '../../../common/shared'
-import { tarExtension, tarGzExtension } from '../constants/compress'
+import { TAR_EXTENSION, TAR_GZ_EXTENSION } from '../constants/compress'
export class CopyMoveFileDto {
@IsNotEmpty()
@@ -58,8 +58,8 @@ export class CompressFileDto {
@IsNotEmpty()
@IsString()
- @IsIn([tarExtension, tarGzExtension])
- extension: typeof tarExtension | typeof tarGzExtension
+ @IsIn([TAR_EXTENSION, TAR_GZ_EXTENSION])
+ extension: typeof TAR_EXTENSION | typeof TAR_GZ_EXTENSION
}
export class SearchFilesDto {
diff --git a/backend/src/applications/files/events/files-events.utils.ts b/backend/src/applications/files/events/files-events.utils.ts
new file mode 100644
index 00000000..9d5b6678
--- /dev/null
+++ b/backend/src/applications/files/events/files-events.utils.ts
@@ -0,0 +1,24 @@
+import { FILE_REPOSITORY } from '../constants/operations'
+import type { SpaceEnv } from '../../spaces/models/space-env.model'
+
+export function SpaceToFileRepository(userId: number, space: SpaceEnv): { id: number; type: FILE_REPOSITORY } | null {
+ if (space.inPersonalSpace) {
+ // Personal user space
+ return { id: userId, type: FILE_REPOSITORY.USER }
+ } else if (space.root?.externalPath) {
+ // External paths used as shares or as space roots share the same quota as their origin
+ if (space.inSharesRepository) {
+ return { id: space.root?.externalParentShareId || space.id, type: FILE_REPOSITORY.SHARE }
+ }
+ return { id: space.id, type: FILE_REPOSITORY.SPACE }
+ } else if (space.root?.file?.path && space.root.owner?.login) {
+ // Space root is linked to a user file
+ return { id: space.root.owner.id, type: FILE_REPOSITORY.USER }
+ } else if (space.root?.file?.space?.id) {
+ return { id: space.root.file.space.id, type: FILE_REPOSITORY.SPACE }
+ } else if (space.id) {
+ return { id: space.id, type: FILE_REPOSITORY.SPACE }
+ } else {
+ return null
+ }
+}
diff --git a/backend/src/applications/files/files.module.ts b/backend/src/applications/files/files.module.ts
index 111875d4..6c8638a5 100644
--- a/backend/src/applications/files/files.module.ts
+++ b/backend/src/applications/files/files.module.ts
@@ -1,12 +1,12 @@
import { Module } from '@nestjs/common'
import { configuration } from '../../configuration/config.environment'
-import { FilesIndexerMySQL } from './adapters/files-indexer-mysql.service'
+import { FilesContentStoreMySQL } from './adapters/files-content-store-mysql.service'
import { FilesTasksController } from './files-tasks.controller'
import { FilesController } from './files.controller'
-import { FilesIndexer } from './models/files-indexer'
+import { FilesContentStore } from './models/files-content-store'
import { CollaboraOnlineModule } from './modules/collabora-online/collabora-online.module'
import { OnlyOfficeModule } from './modules/only-office/only-office.module'
-import { FilesContentManager } from './services/files-content-manager.service'
+import { FilesContentIndexer } from './services/files-content-indexer.service'
import { FilesLockManager } from './services/files-lock-manager.service'
import { FilesManager } from './services/files-manager.service'
import { FilesMethods } from './services/files-methods.service'
@@ -34,8 +34,8 @@ import { FilesQuotaManager } from './services/files-quota-manager.service'
FilesScheduler,
FilesRecents,
FilesParser,
- FilesContentManager,
- { provide: FilesIndexer, useClass: FilesIndexerMySQL },
+ FilesContentIndexer,
+ { provide: FilesContentStore, useClass: FilesContentStoreMySQL },
FilesSearchManager,
FilesEventManager,
FilesQuotaManager
diff --git a/backend/src/applications/files/interfaces/file-event.interface.ts b/backend/src/applications/files/interfaces/file-event.interface.ts
index b5e5cc0a..42bb61dc 100644
--- a/backend/src/applications/files/interfaces/file-event.interface.ts
+++ b/backend/src/applications/files/interfaces/file-event.interface.ts
@@ -12,7 +12,6 @@ export interface FileEventType {
space: SpaceEnv
action: ACTION
rPath: string
- oriPath?: string // src from copy/move operations
}
export interface FileEventEmit {
diff --git a/backend/src/applications/files/models/files-indexer.ts b/backend/src/applications/files/models/files-content-store.ts
similarity index 95%
rename from backend/src/applications/files/models/files-indexer.ts
rename to backend/src/applications/files/models/files-content-store.ts
index 68bbb78c..437bc7ca 100644
--- a/backend/src/applications/files/models/files-indexer.ts
+++ b/backend/src/applications/files/models/files-content-store.ts
@@ -1,6 +1,6 @@
import { FileContent } from '../schemas/file-content.interface'
-export abstract class FilesIndexer {
+export abstract class FilesContentStore {
abstract indexesList(): Promise<string[]>
abstract getIndexName(indexSuffix: string): string
diff --git a/backend/src/applications/files/services/files-content-manager.service.ts b/backend/src/applications/files/services/files-content-indexer.service.ts
similarity index 86%
rename from backend/src/applications/files/services/files-content-manager.service.ts
rename to backend/src/applications/files/services/files-content-indexer.service.ts
index c609aacd..4012e5e3 100644
--- a/backend/src/applications/files/services/files-content-manager.service.ts
+++ b/backend/src/applications/files/services/files-content-indexer.service.ts
@@ -2,28 +2,28 @@ import { Injectable, Logger } from '@nestjs/common'
import fs from 'fs/promises'
import { Stats } from 'node:fs'
import path from 'node:path'
-import { indexableExtensions } from '../constants/indexing'
+import { INDEXABLE_EXTENSIONS } from '../constants/indexing'
import { FileIndexContext, FileParseContext } from '../interfaces/file-parse-index'
-import { FilesIndexer } from '../models/files-indexer'
+import { FilesContentStore } from '../models/files-content-store'
import { FileContent } from '../schemas/file-content.interface'
import { docTextify } from '../utils/doc-textify/doc-textify'
import { PdfOCRWorkerManager } from '../utils/doc-textify/utils/pdf-ocr'
-import { getMimeType } from '../utils/files'
+import { getExtensionWithoutDot, getMimeType } from '../utils/files'
import { FilesParser } from './files-parser.service'
-import { FILE_REPOSITORY } from '../constants/operations'
+import { genIndexingKey } from '../utils/indexing'
@Injectable()
-export class FilesContentManager {
+export class FilesContentIndexer {
private readonly maxDocumentSize = 150 * 1_000_000
- private readonly logger = new Logger(FilesContentManager.name)
+ private readonly logger = new Logger(FilesContentIndexer.name)
private pdfOcrWorkerManager: PdfOCRWorkerManager | null = null
constructor(
- private readonly filesIndexer: FilesIndexer,
+ private readonly filesIndexer: FilesContentStore,
private readonly filesParser: FilesParser
) {}
- async parseAndIndexAllFiles(): Promise<void> {
+ async parseAndIndexAllFiles(userIds?: number[], spaceIds?: number[], shareIds?: number[]): Promise<void> {
this.pdfOcrWorkerManager = PdfOCRWorkerManager.getInstance(this.logger)
try {
await this.pdfOcrWorkerManager.start()
@@ -32,18 +32,8 @@ export class FilesContentManager {
}
try {
const indexSuffixes: string[] = []
- for await (const [id, type, paths] of this.filesParser.allPaths()) {
- let indexSuffix: string
- switch (type) {
- case FILE_REPOSITORY.USER:
- indexSuffix = `${FILE_REPOSITORY.USER}_${id}`
- break
- case FILE_REPOSITORY.SPACE:
- indexSuffix = `${FILE_REPOSITORY.SPACE}_${id}`
- break
- case FILE_REPOSITORY.SHARE:
- indexSuffix = `${FILE_REPOSITORY.SHARE}_${id}`
- }
+ for await (const [id, type, paths] of this.filesParser.allPaths(userIds, spaceIds, shareIds)) {
+ const indexSuffix = genIndexingKey(id, type)
try {
await this.indexFiles(indexSuffix, paths)
} catch (e) {
@@ -51,8 +41,10 @@ export class FilesContentManager {
}
indexSuffixes.push(indexSuffix)
}
- // clean up old tables
- await this.filesIndexer.cleanIndexes(indexSuffixes)
+ // clean up old tables only when all indexes have been indexed
+ if (!userIds?.length && !spaceIds?.length && !shareIds?.length) {
+ await this.filesIndexer.cleanIndexes(indexSuffixes)
+ }
} finally {
await this.pdfOcrWorkerManager?.stop()
this.pdfOcrWorkerManager = null
@@ -142,8 +134,8 @@ export class FilesContentManager {
}
private async analyzeFile(realPath: string, context: FileIndexContext, isRootFile = false): Promise<FileContent> {
- const extension = path.extname(realPath).slice(1).toLowerCase()
- if (!indexableExtensions.has(extension)) return null
+ const extension = getExtensionWithoutDot(realPath)
+ if (!INDEXABLE_EXTENSIONS.has(extension)) return null
const fileName = isRootFile ? path.basename(context.pathPrefix) : path.basename(realPath)
diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.ts
index f85c8f67..dc56c63e 100644
--- a/backend/src/applications/files/services/files-manager.service.ts
+++ b/backend/src/applications/files/services/files-manager.service.ts
@@ -23,7 +23,7 @@ import { canAccessToSpace, haveSpaceEnvPermissions } from '../../spaces/utils/pe
import { UserModel } from '../../users/models/user.model'
import { DEPTH, LOCK_DEPTH } from '../../webdav/constants/webdav'
import { CACHE_LOCK_FILE_TTL } from '../constants/cache'
-import { tarGzExtension } from '../constants/compress'
+import { TAR_GZ_EXTENSION } from '../constants/compress'
import { COMPRESSION_EXTENSION, DEFAULT_HIGH_WATER_MARK } from '../constants/files'
import { FILE_OPERATION } from '../constants/operations'
import { DOCUMENT_TYPE, SAMPLE_PATH_WITHOUT_EXT } from '../constants/samples'
@@ -566,7 +566,7 @@ export class FilesManager {
const archiveExt = dto.name.endsWith(dto.extension) ? '' : `.${dto.extension}`
const dstPath = await uniqueFilePathFromDir(path.join(dto.compressInDirectory ? srcPath : user.tasksPath, `${dto.name}${archiveExt}`))
const archive: Archiver = archiver('tar', {
- gzip: dto.extension === tarGzExtension,
+ gzip: dto.extension === TAR_GZ_EXTENSION,
gzipOptions: {
level: 9
}
@@ -641,7 +641,7 @@ export class FilesManager {
await extractTar({
file: space.realPath,
cwd: dstPath,
- gzip: COMPRESSION_EXTENSION.get(extension) === tarGzExtension,
+ gzip: COMPRESSION_EXTENSION.get(extension) === TAR_GZ_EXTENSION,
preserveOwner: false
})
}
diff --git a/backend/src/applications/files/services/files-methods.service.spec.ts b/backend/src/applications/files/services/files-methods.service.spec.ts
index 3aae7a7d..33fdd35c 100644
--- a/backend/src/applications/files/services/files-methods.service.spec.ts
+++ b/backend/src/applications/files/services/files-methods.service.spec.ts
@@ -14,7 +14,7 @@ import { SpacesQueries } from '../../spaces/services/spaces-queries.service'
import { UserModel } from '../../users/models/user.model'
import { UsersQueries } from '../../users/services/users-queries.service'
import { generateUserTest } from '../../users/utils/test'
-import { tarExtension } from '../constants/compress'
+import { TAR_EXTENSION } from '../constants/compress'
import { CompressFileDto, CopyMoveFileDto } from '../dto/file-operations.dto'
import { FilesManager } from './files-manager.service'
import { FilesMethods } from './files-methods.service'
@@ -97,7 +97,7 @@ describe(FilesMethods.name, () => {
name: '../../archive',
compressInDirectory: false,
files: [{ name: '../../foo', rootAlias: undefined }],
- extension: tarExtension
+ extension: TAR_EXTENSION
}
expect(() => transformAndValidate(CompressFileDto, compressFileDto)).toThrow()
await expect(filesMethods.compress(userTest, spaceEnv, compressFileDto)).rejects.toThrow(/does not exist/i)
diff --git a/backend/src/applications/files/services/files-parser.service.ts b/backend/src/applications/files/services/files-parser.service.ts
index aee1ad26..2364f08e 100644
--- a/backend/src/applications/files/services/files-parser.service.ts
+++ b/backend/src/applications/files/services/files-parser.service.ts
@@ -24,20 +24,20 @@ export class FilesParser {
constructor(@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema) {}
- async *allPaths(userId?: number, spaceIds?: number[], shareIds?: number[]): AsyncGenerator<[number, FILE_REPOSITORY, FileParseContext[]]> {
- yield* this.userPaths(userId)
+ async *allPaths(userIds?: number[], spaceIds?: number[], shareIds?: number[]): AsyncGenerator<[number, FILE_REPOSITORY, FileParseContext[]]> {
+ yield* this.userPaths(userIds)
yield* this.spacePaths(spaceIds)
yield* this.sharePaths(shareIds)
}
- async *userPaths(userId?: number): AsyncGenerator<[number, FILE_REPOSITORY, FileParseContext[]]> {
+ async *userPaths(userIds?: number[]): AsyncGenerator<[number, FILE_REPOSITORY, FileParseContext[]]> {
for (const user of await this.db
.select({
id: users.id,
login: users.login
})
.from(users)
- .where(and(...[eq(users.storageIndexing, true), lte(users.role, USER_ROLE.USER), ...(userId ? [eq(users.id, userId)] : [])]))) {
+ .where(and(...[eq(users.storageIndexing, true), lte(users.role, USER_ROLE.USER), ...(userIds ? [inArray(users.id, userIds)] : [])]))) {
const userFilesPath = UserModel.getFilesPath(user.login)
if (!(await isPathExists(userFilesPath))) {
this.logger.warn({ tag: this.userPaths.name, msg: `user path does not exist : ${userFilesPath}` })
diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.ts
index 2777d449..2bf3cbfc 100644
--- a/backend/src/applications/files/services/files-scheduler.service.ts
+++ b/backend/src/applications/files/services/files-scheduler.service.ts
@@ -17,7 +17,7 @@ import { FileTask, FileTaskStatus } from '../models/file-task'
import { filesRecents } from '../schemas/files-recents.schema'
import { files } from '../schemas/files.schema'
import { dirHasChildren, isPathExists, removeFiles } from '../utils/files'
-import { FilesContentManager } from './files-content-manager.service'
+import { FilesContentIndexer } from './files-content-indexer.service'
import { FilesTasksManager } from './files-tasks-manager.service'
import { FilesQuotaManager } from './files-quota-manager.service'
@@ -31,7 +31,7 @@ export class FilesScheduler {
constructor(
@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema,
private readonly cache: Cache,
- private readonly filesContentManager: FilesContentManager,
+ private readonly filesContentIndexer: FilesContentIndexer,
private readonly filesQuotaManager: FilesQuotaManager
) {}
@@ -46,7 +46,7 @@ export class FilesScheduler {
}
}
- @Timeout(180_000)
+ @Timeout(300_000)
async afterStartup(): Promise<void> {
try {
await this.indexContentFiles()
@@ -161,7 +161,7 @@ export class FilesScheduler {
this.isIndexContentFilesRunning = true
this.logger.log({ tag: this.indexContentFiles.name, msg: `START` })
try {
- await this.filesContentManager.parseAndIndexAllFiles()
+ await this.filesContentIndexer.parseAndIndexAllFiles()
this.logger.log({ tag: this.indexContentFiles.name, msg: `END` })
} finally {
this.isIndexContentFilesRunning = false
diff --git a/backend/src/applications/files/services/files-search-manager.service.spec.ts b/backend/src/applications/files/services/files-search-manager.service.spec.ts
index 95b99383..cf9f416a 100644
--- a/backend/src/applications/files/services/files-search-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-search-manager.service.spec.ts
@@ -1,7 +1,7 @@
import { Test, TestingModule } from '@nestjs/testing'
import { SharesQueries } from '../../shares/services/shares-queries.service'
import { SpacesQueries } from '../../spaces/services/spaces-queries.service'
-import { FilesIndexer } from '../models/files-indexer'
+import { FilesContentStore } from '../models/files-content-store'
import { FilesParser } from './files-parser.service'
import { FilesSearchManager } from './files-search-manager.service'
@@ -12,7 +12,7 @@ describe(FilesSearchManager.name, () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
FilesSearchManager,
- { provide: FilesIndexer, useValue: {} },
+ { provide: FilesContentStore, useValue: {} },
{ provide: FilesParser, useValue: {} },
{
provide: SpacesQueries,
diff --git a/backend/src/applications/files/services/files-search-manager.service.ts b/backend/src/applications/files/services/files-search-manager.service.ts
index 765029d5..b6f65a23 100644
--- a/backend/src/applications/files/services/files-search-manager.service.ts
+++ b/backend/src/applications/files/services/files-search-manager.service.ts
@@ -7,19 +7,20 @@ import { SharesQueries } from '../../shares/services/shares-queries.service'
import { SpacesQueries } from '../../spaces/services/spaces-queries.service'
import { UserModel } from '../../users/models/user.model'
import { SearchFilesDto } from '../dto/file-operations.dto'
-import { FilesIndexer } from '../models/files-indexer'
+import { FilesContentStore } from '../models/files-content-store'
import { FileContent } from '../schemas/file-content.interface'
import { dirName, fileName, getMimeType } from '../utils/files'
import { genRegexPositiveAndNegativeTerms } from '../utils/files-search'
import { FilesParser } from './files-parser.service'
import { FILE_REPOSITORY } from '../constants/operations'
+import { genIndexingKey } from '../utils/indexing'
@Injectable()
export class FilesSearchManager {
private readonly logger = new Logger(FilesSearchManager.name)
constructor(
- private readonly filesIndexer: FilesIndexer,
+ private readonly filesIndexer: FilesContentStore,
private readonly filesParser: FilesParser,
private readonly spacesQueries: SpacesQueries,
private readonly sharesQueries: SharesQueries
@@ -39,9 +40,9 @@ export class FilesSearchManager {
private async searchFullText(userId: number, spaceIds: number[], shareIds: number[], search: string, limit: number): Promise<FileContent[]> {
const indexNames = await this.filesIndexer.existingIndexes([
- `${FILE_REPOSITORY.USER}_${userId}`,
- ...spaceIds.map((id) => `${FILE_REPOSITORY.SPACE}_${id}`),
- ...shareIds.map((id) => `${FILE_REPOSITORY.SHARE}_${id}`)
+ genIndexingKey(userId, FILE_REPOSITORY.USER),
+ ...spaceIds.map((id) => genIndexingKey(id, FILE_REPOSITORY.SPACE)),
+ ...shareIds.map((id) => genIndexingKey(id, FILE_REPOSITORY.SHARE))
])
if (indexNames.length === 0) {
return []
@@ -63,7 +64,7 @@ export class FilesSearchManager {
private async searchFileNames(userId: number, spaceIds: number[], shareIds: number[], search: string, limit: number): Promise<FileContent[]> {
const fileContents: FileContent[] = []
const regexpTerms = genRegexPositiveAndNegativeTerms(search)
- for await (const [_id, _type, paths] of this.filesParser.allPaths(userId, spaceIds, shareIds)) {
+ for await (const [_id, _type, paths] of this.filesParser.allPaths([userId], spaceIds, shareIds)) {
for (const p of paths) {
const regexBasePath = new RegExp(`^/?${p.realPath}/?`)
if (!p.isDir) {
diff --git a/backend/src/applications/files/utils/doc-textify/doc-textify.ts b/backend/src/applications/files/utils/doc-textify/doc-textify.ts
index df65c5eb..27bd8fa9 100644
--- a/backend/src/applications/files/utils/doc-textify/doc-textify.ts
+++ b/backend/src/applications/files/utils/doc-textify/doc-textify.ts
@@ -1,5 +1,4 @@
import fs from 'node:fs/promises'
-import path from 'node:path'
import { parseExcel } from './adapters/excel'
import { parseHtml } from './adapters/html'
import { parseMarkdown } from './adapters/markdown'
@@ -10,6 +9,8 @@ import { parseText } from './adapters/text'
import { parseWord } from './adapters/word'
import { DocTextifyOptions } from './interfaces/doc-textify.interfaces'
import { cleanContent } from './utils/clean'
+import { INDEXABLE_EXTENSIONS } from '../../constants/indexing'
+import { getExtensionWithoutDot } from '../files'
/** Main: determine parser by extension and dispatch */
export async function docTextify(
@@ -31,7 +32,11 @@ export async function docTextify(
}
}
- const ext = fileProperties?.extension || path.extname(filePath).slice(1).toLowerCase()
+ const ext = fileProperties?.extension || getExtensionWithoutDot(filePath)
+
+ if (!INDEXABLE_EXTENSIONS.has(ext)) {
+ throw new Error(`currently only supports ${[...INDEXABLE_EXTENSIONS].join(',')} files`)
+ }
switch (ext) {
case 'docx':
@@ -54,6 +59,6 @@ export async function docTextify(
case 'htm':
return cleanContent(await parseHtml(filePath), options)
default:
- throw new Error('currently only supports docx, pptx, xlsx, odt, odp, ods, pdf, txt, md, html files')
+ throw new Error(`no handler found for extension: ${ext}`)
}
}
diff --git a/backend/src/applications/files/utils/files-search.ts b/backend/src/applications/files/utils/files-search.ts
index b9dab0ee..d42d3d8c 100644
--- a/backend/src/applications/files/utils/files-search.ts
+++ b/backend/src/applications/files/utils/files-search.ts
@@ -1,5 +1,5 @@
import { escapeString } from '../../../common/functions'
-import { minCharsToSearch } from '../constants/indexing'
+import { MIN_CHARS_TO_SEARCH } from '../constants/indexing'
const regexMatchSearchBoolean = new RegExp(`([+-]?)(?:"([^"]+)"|(\\S+))`)
const regexMatchesSearchBoolean = new RegExp(regexMatchSearchBoolean.source, 'g')
@@ -59,7 +59,7 @@ export function analyzeTerms(search: string, onlyAllowNegative = false): string[
const [, operator, quoted, unquoted] = match.match(regexMatchSearchBoolean)
let term: string = (quoted || unquoted).trim()
- if (term.length < minCharsToSearch) return null
+ if (term.length < MIN_CHARS_TO_SEARCH) return null
if ((onlyAllowNegative && operator !== '-') || (!onlyAllowNegative && (operator === '-' || operator === '~'))) return null
diff --git a/backend/src/applications/files/utils/files.ts b/backend/src/applications/files/utils/files.ts
index a143a3ef..841603e7 100644
--- a/backend/src/applications/files/utils/files.ts
+++ b/backend/src/applications/files/utils/files.ts
@@ -97,6 +97,10 @@ export function getMimeType(fPath: string, isDir: boolean): string {
return 'file'
}
+export function getExtensionWithoutDot(fPath: string): string {
+ return path.extname(fPath).slice(1).toLowerCase()
+}
+
export function genEtag(file?: Pick<FileProps, 'size' | 'mtime'>, rPath?: string, weakPrefix = true): string {
if (!file) {
if (!rPath) throw new Error('File or path are missing')
diff --git a/backend/src/applications/files/utils/indexing.ts b/backend/src/applications/files/utils/indexing.ts
new file mode 100644
index 00000000..c2190f8e
--- /dev/null
+++ b/backend/src/applications/files/utils/indexing.ts
@@ -0,0 +1,15 @@
+import { FILE_REPOSITORY } from '../constants/operations'
+import { CACHE_INDEXING_UPDATE_PREFIX } from '../constants/indexing'
+import { SpaceEnv } from '../../spaces/models/space-env.model'
+import { SpaceToFileRepository } from '../events/files-events.utils'
+
+export function genIndexingKey(id: number, type: FILE_REPOSITORY, sep = '_'): string {
+ return `${type}${sep}${id}`
+}
+
+export function indexingUpdateCacheKeyFromSpace(userId: number, space: SpaceEnv): string | null {
+ const repository: { id: number; type: FILE_REPOSITORY } = SpaceToFileRepository(userId, space)
+ if (repository === null) return null
+ const indexingKey = genIndexingKey(repository.id, repository.type, '-')
+ return `${CACHE_INDEXING_UPDATE_PREFIX}-${indexingKey}`
+}
diff --git a/backend/src/applications/files/utils/quota.ts b/backend/src/applications/files/utils/quota.ts
index 3f8093e9..bf3b76a9 100644
--- a/backend/src/applications/files/utils/quota.ts
+++ b/backend/src/applications/files/utils/quota.ts
@@ -1,29 +1,13 @@
import { SpaceEnv } from '../../spaces/models/space-env.model'
import { CACHE_QUOTA_EVENT_UPDATE_PREFIX, CACHE_QUOTA_PREFIX } from '../constants/cache'
import { FILE_REPOSITORY } from '../constants/operations'
+import { SpaceToFileRepository } from '../events/files-events.utils'
export function genQuotaCacheKey(id: number, type: FILE_REPOSITORY, isEventUpdate = false): string {
return `${isEventUpdate ? CACHE_QUOTA_EVENT_UPDATE_PREFIX : CACHE_QUOTA_PREFIX}-${type}-${id}`
}
-export function quotaCacheKeyFromSpace(userId: number, space: SpaceEnv, isEventUpdate = false): string {
- if (space.inPersonalSpace) {
- // Personal user space
- return genQuotaCacheKey(userId, FILE_REPOSITORY.USER, isEventUpdate)
- } else if (space.root?.externalPath) {
- // External paths used as shares or as space roots share the same quota as their origin
- if (space.inSharesRepository) {
- return genQuotaCacheKey(space.root?.externalParentShareId || space.id, FILE_REPOSITORY.SHARE, isEventUpdate)
- }
- return genQuotaCacheKey(space.id, FILE_REPOSITORY.SPACE, isEventUpdate)
- } else if (space.root?.file?.path && space.root.owner?.login) {
- // Space root is linked to a user file
- return genQuotaCacheKey(space.root.owner.id, FILE_REPOSITORY.USER, isEventUpdate)
- } else if (space.root?.file?.space?.id) {
- return genQuotaCacheKey(space.root.file.space.id, FILE_REPOSITORY.SPACE, isEventUpdate)
- } else if (space.id) {
- return genQuotaCacheKey(space.id, FILE_REPOSITORY.SPACE, isEventUpdate)
- } else {
- return null
- }
+export function quotaCacheKeyFromSpace(userId: number, space: SpaceEnv, isEventUpdate = false): string | null {
+ const repository: { id: number; type: FILE_REPOSITORY } = SpaceToFileRepository(userId, space)
+ return repository === null ? null : genQuotaCacheKey(repository.id, repository.type, isEventUpdate)
}
diff --git a/frontend/src/app/applications/files/components/dialogs/files-compression-dialog.component.ts b/frontend/src/app/applications/files/components/dialogs/files-compression-dialog.component.ts
index c3df4244..0b15df6b 100644
--- a/frontend/src/app/applications/files/components/dialogs/files-compression-dialog.component.ts
+++ b/frontend/src/app/applications/files/components/dialogs/files-compression-dialog.component.ts
@@ -2,7 +2,7 @@ import { ChangeDetectionStrategy, Component, EventEmitter, HostListener, inject,
import { FormsModule } from '@angular/forms'
import { FaIconComponent } from '@fortawesome/angular-fontawesome'
import { faFileArchive } from '@fortawesome/free-solid-svg-icons'
-import { tarExtension, tarGzExtension } from '@sync-in-server/backend/src/applications/files/constants/compress'
+import { TAR_EXTENSION, TAR_GZ_EXTENSION } from '@sync-in-server/backend/src/applications/files/constants/compress'
import type { CompressFileDto } from '@sync-in-server/backend/src/applications/files/dto/file-operations.dto'
import { L10N_LOCALE, L10nLocale, L10nTranslateDirective, L10nTranslatePipe } from 'angular-l10n'
import { AutofocusDirective } from '../../../../common/directives/auto-focus.directive'
@@ -16,7 +16,7 @@ import { FilesService } from '../../services/files.service'
changeDetection: ChangeDetectionStrategy.OnPush
})
export class FilesCompressionDialogComponent implements OnInit {
- @Input() archiveProps: CompressFileDto = { name: '', files: [], compressInDirectory: true, extension: tarExtension }
+ @Input() archiveProps: CompressFileDto = { name: '', files: [], compressInDirectory: true, extension: TAR_EXTENSION }
@Output() submitEvent = new EventEmitter()
public disableInDirCompression = false
protected readonly locale = inject<L10nLocale>(L10N_LOCALE)
@@ -47,6 +47,6 @@ export class FilesCompressionDialogComponent implements OnInit {
}
setCompression(enable: boolean) {
- this.archiveProps.extension = enable ? tarGzExtension : tarExtension
+ this.archiveProps.extension = enable ? TAR_GZ_EXTENSION : TAR_EXTENSION
}
}
diff --git a/frontend/src/app/applications/files/components/sidebar/files-clipboard.component.ts b/frontend/src/app/applications/files/components/sidebar/files-clipboard.component.ts
index c1e16867..f5fc7c18 100644
--- a/frontend/src/app/applications/files/components/sidebar/files-clipboard.component.ts
+++ b/frontend/src/app/applications/files/components/sidebar/files-clipboard.component.ts
@@ -3,7 +3,7 @@ import { Component, inject, OnDestroy } from '@angular/core'
import { FormsModule } from '@angular/forms'
import { FaIconComponent } from '@fortawesome/angular-fontawesome'
import { faArrowsAlt, faClone, faDownload, faQuestion, faTimes, faTrashCan } from '@fortawesome/free-solid-svg-icons'
-import { tarExtension } from '@sync-in-server/backend/src/applications/files/constants/compress'
+import { TAR_EXTENSION } from '@sync-in-server/backend/src/applications/files/constants/compress'
import { FILE_OPERATION } from '@sync-in-server/backend/src/applications/files/constants/operations'
import type { CompressFileDto } from '@sync-in-server/backend/src/applications/files/dto/file-operations.dto'
import { L10N_LOCALE, L10nLocale, L10nTranslateDirective, L10nTranslatePipe } from 'angular-l10n'
@@ -70,7 +70,7 @@ export class FilesClipboardComponent implements OnDestroy {
name: this.files[0].name,
compressInDirectory: this.operations[this.selectedAction].operation === FILE_OPERATION.COMPRESS,
files: this.files.map((f: FileModel) => ({ name: f.name, rootAlias: f.root?.alias, path: f.path })),
- extension: tarExtension
+ extension: TAR_EXTENSION
}
const modalRef: BsModalRef<FilesCompressionDialogComponent> = this.layout.openDialog(FilesCompressionDialogComponent, null, {
initialState: { archiveProps: archiveProps } as FilesCompressionDialogComponent
diff --git a/frontend/src/app/applications/search/components/search.component.ts b/frontend/src/app/applications/search/components/search.component.ts
index 204e7cd6..91a995c1 100644
--- a/frontend/src/app/applications/search/components/search.component.ts
+++ b/frontend/src/app/applications/search/components/search.component.ts
@@ -4,7 +4,7 @@ import { FormsModule } from '@angular/forms'
import { Router } from '@angular/router'
import { FaIconComponent } from '@fortawesome/angular-fontawesome'
import { faFont, faMapMarkerAlt, faSpinner, faTimes, faTrashCan } from '@fortawesome/free-solid-svg-icons'
-import { minCharsToSearch } from '@sync-in-server/backend/src/applications/files/constants/indexing'
+import { MIN_CHARS_TO_SEARCH } from '@sync-in-server/backend/src/applications/files/constants/indexing'
import type { SearchFilesDto } from '@sync-in-server/backend/src/applications/files/dto/file-operations.dto'
import { L10N_LOCALE, L10nLocale, L10nTranslateDirective, L10nTranslatePipe } from 'angular-l10n'
import { ButtonCheckboxDirective } from 'ngx-bootstrap/buttons'
@@ -45,7 +45,7 @@ export class SearchComponent {
protected readonly store = inject(StoreService)
public searchContent: Signal<string> = computed(() => this.store.currentSearch().content)
protected readonly icons = { SEARCH_ICON, faSpinner, faTrashCan, faTimes, faFont, faMapMarkerAlt }
- protected minCharsToSearch = minCharsToSearch
+ protected minCharsToSearch = MIN_CHARS_TO_SEARCH
protected loading = false
protected errorMessage: string = null
protected selectedId: number = null
diff --git a/frontend/src/app/applications/spaces/components/spaces-browser.component.ts b/frontend/src/app/applications/spaces/components/spaces-browser.component.ts
index e41304f9..3855e93a 100644
--- a/frontend/src/app/applications/spaces/components/spaces-browser.component.ts
+++ b/frontend/src/app/applications/spaces/components/spaces-browser.component.ts
@@ -31,7 +31,7 @@ import {
faUpload
} from '@fortawesome/free-solid-svg-icons'
import { ContextMenuComponent, ContextMenuModule } from '@perfectmemory/ngx-contextmenu'
-import { tarExtension } from '@sync-in-server/backend/src/applications/files/constants/compress'
+import { TAR_EXTENSION } from '@sync-in-server/backend/src/applications/files/constants/compress'
import { FILE_OPERATION } from '@sync-in-server/backend/src/applications/files/constants/operations'
import type { CompressFileDto } from '@sync-in-server/backend/src/applications/files/dto/file-operations.dto'
import type { FileProps } from '@sync-in-server/backend/src/applications/files/interfaces/file-props.interface'
@@ -609,7 +609,7 @@ export class SpacesBrowserComponent implements OnInit, AfterViewInit, OnDestroy
name: this.selection[0].name,
compressInDirectory: this.inSharesList ? false : inDir,
files: this.selection.map((f: FileModel) => ({ name: f.name, rootAlias: f.root?.alias })),
- extension: tarExtension
+ extension: TAR_EXTENSION
}
this.layout.openDialog(FilesCompressionDialogComponent, null, {
initialState: {