Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
feat(backend:files): add support for configurable OCR language paths
johaven48443aa8578c
diff --git a/backend/src/applications/files/files.config.ts b/backend/src/applications/files/files.config.tsindex bf2e99fc..ea7c1341 100644--- a/backend/src/applications/files/files.config.ts+++ b/backend/src/applications/files/files.config.ts@@ -1,5 +1,16 @@import { Type } from 'class-transformer'-import { ArrayNotEmpty, IsArray, IsBoolean, IsInt, IsNotEmpty, IsNotEmptyObject, IsString, ValidateIf, ValidateNested } from 'class-validator'+import {+ ArrayNotEmpty,+ IsArray,+ IsBoolean,+ IsInt,+ IsNotEmpty,+ IsNotEmptyObject,+ IsOptional,+ IsString,+ ValidateIf,+ ValidateNested+} from 'class-validator'import { CollaboraOnlineConfig } from './modules/collabora-online/collabora-online.config'import { OnlyOfficeConfig } from './modules/only-office/only-office.config'@@ -15,6 +26,10 @@ export class FilesContentIndexingOCRConfig {@IsBoolean()offline: boolean = false++ @IsOptional()+ @IsString()+ languagesPath?: string}export class FilesContentIndexingConfig {diff --git a/backend/src/applications/files/services/files-content-indexer.service.spec.ts b/backend/src/applications/files/services/files-content-indexer.service.spec.tsindex f9adbdae..0fcbf85b 100644--- a/backend/src/applications/files/services/files-content-indexer.service.spec.ts+++ b/backend/src/applications/files/services/files-content-indexer.service.spec.ts@@ -7,7 +7,7 @@ import { FILE_REPOSITORY } from '../constants/operations'import { FileParseContext } from '../interfaces/file-parse-index'import { FilesContentStore } from '../models/files-content-store'import * as docTextifyModule from '../utils/doc-textify/doc-textify'-import { PdfOCRWorkerManager } from '../utils/doc-textify/utils/pdf-ocr'+import { OCRManager } from '../utils/doc-textify/utils/ocr'import { FilesParser } from './files-parser.service'import { FilesContentIndexer } from './files-content-indexer.service'@@ -111,7 +111,7 @@ describe(FilesContentIndexer.name, () => {start: jest.fn().mockResolvedValue(null),stop: jest.fn().mockResolvedValue(undefined)}- jest.spyOn(PdfOCRWorkerManager, 'getInstance').mockReturnValue(ocrManager as any)+ jest.spyOn(OCRManager, 'getInstance').mockReturnValue(ocrManager as any)filesParser.allPaths.mockReturnValue(asyncGen([[1, FILE_REPOSITORY.USER, [{ realPath: '/u/john', pathPrefix: 'files/personal', isDir: true }]],@@ -135,7 +135,7 @@ describe(FilesContentIndexer.name, () => {start: jest.fn().mockRejectedValue(new Error('ocr init failed')),stop: jest.fn().mockResolvedValue(undefined)}- jest.spyOn(PdfOCRWorkerManager, 'getInstance').mockReturnValue(ocrManager as any)+ jest.spyOn(OCRManager, 'getInstance').mockReturnValue(ocrManager as any)filesParser.allPaths.mockReturnValue(asyncGen([[9, FILE_REPOSITORY.USER, [{ realPath: '/u/jane', pathPrefix: 'files/personal', isDir: true }]]] as any))@@ -282,7 +282,7 @@ describe(FilesContentIndexer.name, () => {})it('should parse content and return null when parser returns empty or throws', async () => {- ;(service as any).pdfOcrWorkerManager = { worker: { id: 'worker-1' } }+ ;(service as any).ocrManager = { worker: { id: 'worker-1' } }const docTextifySpy = jest.spyOn(docTextifyModule, 'docTextify')docTextifySpy.mockResolvedValueOnce('')docTextifySpy.mockRejectedValueOnce(new Error('parse failed'))diff --git a/backend/src/applications/files/services/files-content-indexer.service.ts b/backend/src/applications/files/services/files-content-indexer.service.tsindex 1f3e6622..d7863b44 100644--- a/backend/src/applications/files/services/files-content-indexer.service.ts+++ b/backend/src/applications/files/services/files-content-indexer.service.ts@@ -7,7 +7,7 @@ import { FileIndexContext, FileParseContext } from '../interfaces/file-parse-indimport { FilesContentStore } from '../models/files-content-store'import { FileContent } from '../schemas/file-content.interface'import { docTextify } from '../utils/doc-textify/doc-textify'-import { PdfOCRWorkerManager } from '../utils/doc-textify/utils/pdf-ocr'+import { OCRManager } from '../utils/doc-textify/utils/ocr'import { getExtensionWithoutDot, getMimeType } from '../utils/files'import { FilesParser } from './files-parser.service'import { genIndexingKey } from '../utils/indexing'@@ -18,7 +18,7 @@ import { Cache } from '../../../infrastructure/cache/services/cache.service'export class FilesContentIndexer {private readonly maxDocumentSize = 150 * 1_000_000private readonly logger = new Logger(FilesContentIndexer.name)- private pdfOcrWorkerManager: PdfOCRWorkerManager | null = null+ private ocrManager: OCRManager | null = nullconstructor(private readonly cache: Cache,@@ -62,9 +62,9 @@ export class FilesContentIndexer {}async parseAndIndexAllFiles(userIds?: number[], spaceIds?: number[], shareIds?: number[]): Promise<void> {- this.pdfOcrWorkerManager = PdfOCRWorkerManager.getInstance(this.logger)+ this.ocrManager = OCRManager.getInstance(this.logger)try {- await this.pdfOcrWorkerManager.start()+ await this.ocrManager.start()} catch (e) {this.logger.warn({ tag: this.parseAndIndexAllFiles.name, msg: `unable to initialize OCR worker: ${e}` })}@@ -84,8 +84,8 @@ export class FilesContentIndexer {await this.filesIndexer.cleanIndexes(indexSuffixes)}} finally {- await this.pdfOcrWorkerManager?.stop()- this.pdfOcrWorkerManager = null+ await this.ocrManager?.stop()+ this.ocrManager = null}}@@ -147,7 +147,7 @@ export class FilesContentIndexer {} else {this.logger.log({tag: this.indexFiles.name,- msg: `${indexSuffix} - indexed: ${indexedRecords - errorRecords}, errors: ${errorRecords}, deleted: ${recordsToDelete.length}`+ msg: `${indexSuffix} - indexed: ${indexedRecords - errorRecords}, deleted: ${recordsToDelete.length}, errors: ${errorRecords}`})}}@@ -224,7 +224,7 @@ export class FilesContentIndexer {{newlineDelimiter: ' ',minCharsToExtract: 10,- ocrWorker: this.pdfOcrWorkerManager?.worker+ ocrWorker: this.ocrManager?.worker},{extension: extension,diff --git a/backend/src/applications/files/services/files-event-manager.service.ts b/backend/src/applications/files/services/files-event-manager.service.tsindex ec2c7c47..6ca7734a 100644--- a/backend/src/applications/files/services/files-event-manager.service.ts+++ b/backend/src/applications/files/services/files-event-manager.service.ts@@ -37,12 +37,12 @@ export class FilesEventManager implements OnModuleDestroy {await this.flushEvents()}- private readonly onFileEvent = (fileEvent: FileEventType): void => {+ private readonly onFileEvent = (fEvent: FileEventType): void => {this.logger.verbose({tag: this.onFileEvent.name,- msg: `Receiving: user:${fileEvent.user.login} action:${fileEvent.action} on ${fileEvent.space.alias}`+ msg: `Receiving: user:${fEvent.user.login} action:${fEvent.action} url:${fEvent.space.url}`})- this.eventsBuffer.push(fileEvent)+ this.eventsBuffer.push(fEvent)if (this.eventsBuffer.length >= this.MAX_BUFFER_SIZE) {void this.flushEvents()returndiff --git a/backend/src/applications/files/utils/doc-textify/utils/pdf-ocr.ts b/backend/src/applications/files/utils/doc-textify/utils/ocr.tssimilarity index 63%rename from backend/src/applications/files/utils/doc-textify/utils/pdf-ocr.tsrename to backend/src/applications/files/utils/doc-textify/utils/ocr.tsindex 69a23040..bc204bb3 100644--- a/backend/src/applications/files/utils/doc-textify/utils/pdf-ocr.ts+++ b/backend/src/applications/files/utils/doc-textify/utils/ocr.ts@@ -3,19 +3,21 @@ import path from 'node:path'import Tesseract from 'tesseract.js'import { configuration } from '../../../../../configuration/config.environment'import { makeDir } from '../../files'+import type { FilesContentIndexingOCRConfig } from '../../../files.config'-export type PdfOCRWorker = Awaited<ReturnType<typeof Tesseract.createWorker>>-type PdfOCRWorkerOptions = NonNullable<Parameters<typeof Tesseract.createWorker>[2]>+type OCRWorker = Awaited<ReturnType<typeof Tesseract.createWorker>>+type OCRWorkerOptions = NonNullable<Parameters<typeof Tesseract.createWorker>[2]>-export class PdfOCRWorkerManager {- private static instance: PdfOCRWorkerManager- public worker: PdfOCRWorker | null- private readonly ocrLanguagesPath = path.resolve(__dirname, '../../../assets/ocr-languages')+export class OCRManager {+ private static instance: OCRManager+ public worker: OCRWorker | null+ private readonly ocrConfig: FilesContentIndexingOCRConfig = configuration.applications.files.contentIndexing.ocr+ private readonly ocrLanguagesPath = this.ocrConfig.languagesPath ?? path.resolve(__dirname, '../../../assets/ocr-languages')private readonly ocrUserDefinedDpi = '300'private readonly ocrTrainedDataExtension = '.traineddata'private readonly ocrTrainedDataGzipExtension = '.traineddata.gz'private logger: Logger- private workerInitializationPromise: Promise<PdfOCRWorker | null> | null+ private workerInitializationPromise: Promise<OCRWorker | null> | nullprivate constructor(logger: Logger) {this.logger = logger@@ -23,24 +25,25 @@ export class PdfOCRWorkerManager {this.workerInitializationPromise = null}- static getInstance(logger: Logger): PdfOCRWorkerManager {- if (!PdfOCRWorkerManager.instance) {- PdfOCRWorkerManager.instance = new PdfOCRWorkerManager(logger)+ static getInstance(logger: Logger): OCRManager {+ if (!OCRManager.instance) {+ OCRManager.instance = new OCRManager(logger)}- return PdfOCRWorkerManager.instance+ return OCRManager.instance}- async start(): Promise<PdfOCRWorker | null> {+ async start(): Promise<OCRWorker | null> {if (this.worker) {return this.worker}if (this.workerInitializationPromise) {return this.workerInitializationPromise}- this.logger.verbose({ tag: this.constructor.name, msg: 'Starting OCR' })+this.workerInitializationPromise = this.createWorkerFromConfiguration().then((worker) => {this.worker = worker+ if (this.worker !== null) this.logger.verbose({ tag: this.constructor.name, msg: 'Started' })return worker}).finally(() => {@@ -50,7 +53,6 @@ export class PdfOCRWorkerManager {}async stop(): Promise<void> {- this.logger.verbose({ tag: this.constructor.name, msg: 'Stopping OCR' })if (this.workerInitializationPromise) {await this.workerInitializationPromise.catch((e) =>this.logger.error({@@ -64,15 +66,19 @@ export class PdfOCRWorkerManager {if (!worker) {return}- await worker.terminate().catch((e) =>- this.logger.error({- tag: this.constructor.name,- msg: `${e}`- })- )++ await worker+ .terminate()+ .then(() => this.logger.verbose({ tag: this.constructor.name, msg: 'Stopped' }))+ .catch((e) =>+ this.logger.error({+ tag: this.constructor.name,+ msg: `${e}`+ })+ )}- private async createConfiguredWorker(languages: string[], options: PdfOCRWorkerOptions): Promise<PdfOCRWorker> {+ private async createConfiguredWorker(languages: string[], options: OCRWorkerOptions): Promise<OCRWorker> {const worker = await Tesseract.createWorker(languages, Tesseract.OEM.LSTM_ONLY, {...options,errorHandler: (e: unknown) =>@@ -90,9 +96,8 @@ export class PdfOCRWorkerManager {return worker}- private async createWorkerFromConfiguration(): Promise<PdfOCRWorker | null> {- const ocrOptions = configuration.applications.files.contentIndexing.ocr- if (!ocrOptions.enabled) {+ private async createWorkerFromConfiguration(): Promise<OCRWorker | null> {+ if (!this.ocrConfig.enabled) {return null}try {@@ -103,14 +108,14 @@ export class PdfOCRWorkerManager {msg: `unable to create languages directory: ${e}`})}- if (ocrOptions.offline) {+ if (this.ocrConfig.offline) {const offlineWorkerOptions = {langPath: this.ocrLanguagesPath,cacheMethod: 'none' as const}try {// First try for gzipped files: <lang>.traineddata.gz- return await this.createConfiguredWorker(ocrOptions.languages, {+ return await this.createConfiguredWorker(this.ocrConfig.languages, {...offlineWorkerOptions,gzip: true})@@ -120,13 +125,13 @@ export class PdfOCRWorkerManager {msg: `unable to load offline OCR languages as ${this.ocrTrainedDataGzipExtension}, retrying with ${this.ocrTrainedDataExtension}: ${error}`})// Fallback for non-gz files: <lang>.traineddata- return this.createConfiguredWorker(ocrOptions.languages, {+ return this.createConfiguredWorker(this.ocrConfig.languages, {...offlineWorkerOptions,gzip: false})}}- return this.createConfiguredWorker(ocrOptions.languages, {+ return this.createConfiguredWorker(this.ocrConfig.languages, {cachePath: this.ocrLanguagesPath,cacheMethod: 'write' as const})diff --git a/environment/environment.dist.yaml b/environment/environment.dist.yamlindex 78a4ad6b..9e76cf28 100755--- a/environment/environment.dist.yaml+++ b/environment/environment.dist.yaml@@ -131,6 +131,9 @@ auth:# ca: [/app/certs/ca.pem]# optionaltlsOptions:+ # baseDN: Distinguished name (e.g.: ou=people,dc=ldap,dc=sync-in,dc=com)+ # Used as the search base for users, and for groups when adminGroup is a CN.+ # requiredbaseDN: ou=people,dc=ldap,dc=sync-in,dc=com# filter, e.g: (acl=admin)# Appended as-is to the LDAP search filter (trusted config).@@ -314,6 +317,10 @@ applications:# To download languages, use this scheme: https://cdn.jsdelivr.net/npm/@tesseract.js-data/<lang>@1.0.0/4.0.0_best_int/<lang>.traineddata.gz# default: falseoffline: false+ # Path to local OCR language files+ # Used when offline mode is enabled or to override default location+ # default: built-in OCR directory+ languagesPath:# Show files starting with a dot in the file explorer# default: falseshowHiddenFiles: false