Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

feat(backend:files): add support for configurable OCR language paths

johaven48443aa8578c
diff --git a/backend/src/applications/files/files.config.ts b/backend/src/applications/files/files.config.ts
index bf2e99fc..ea7c1341 100644
--- a/backend/src/applications/files/files.config.ts
+++ b/backend/src/applications/files/files.config.ts
@@ -1,5 +1,16 @@
import { Type } from 'class-transformer'
-import { ArrayNotEmpty, IsArray, IsBoolean, IsInt, IsNotEmpty, IsNotEmptyObject, IsString, ValidateIf, ValidateNested } from 'class-validator'
+import {
+ ArrayNotEmpty,
+ IsArray,
+ IsBoolean,
+ IsInt,
+ IsNotEmpty,
+ IsNotEmptyObject,
+ IsOptional,
+ IsString,
+ ValidateIf,
+ ValidateNested
+} from 'class-validator'
import { CollaboraOnlineConfig } from './modules/collabora-online/collabora-online.config'
import { OnlyOfficeConfig } from './modules/only-office/only-office.config'
@@ -15,6 +26,10 @@ export class FilesContentIndexingOCRConfig {
@IsBoolean()
offline: boolean = false
+
+ @IsOptional()
+ @IsString()
+ languagesPath?: string
}
export class FilesContentIndexingConfig {
diff --git a/backend/src/applications/files/services/files-content-indexer.service.spec.ts b/backend/src/applications/files/services/files-content-indexer.service.spec.ts
index f9adbdae..0fcbf85b 100644
--- a/backend/src/applications/files/services/files-content-indexer.service.spec.ts
+++ b/backend/src/applications/files/services/files-content-indexer.service.spec.ts
@@ -7,7 +7,7 @@ import { FILE_REPOSITORY } from '../constants/operations'
import { FileParseContext } from '../interfaces/file-parse-index'
import { FilesContentStore } from '../models/files-content-store'
import * as docTextifyModule from '../utils/doc-textify/doc-textify'
-import { PdfOCRWorkerManager } from '../utils/doc-textify/utils/pdf-ocr'
+import { OCRManager } from '../utils/doc-textify/utils/ocr'
import { FilesParser } from './files-parser.service'
import { FilesContentIndexer } from './files-content-indexer.service'
@@ -111,7 +111,7 @@ describe(FilesContentIndexer.name, () => {
start: jest.fn().mockResolvedValue(null),
stop: jest.fn().mockResolvedValue(undefined)
}
- jest.spyOn(PdfOCRWorkerManager, 'getInstance').mockReturnValue(ocrManager as any)
+ jest.spyOn(OCRManager, 'getInstance').mockReturnValue(ocrManager as any)
filesParser.allPaths.mockReturnValue(
asyncGen([
[1, FILE_REPOSITORY.USER, [{ realPath: '/u/john', pathPrefix: 'files/personal', isDir: true }]],
@@ -135,7 +135,7 @@ describe(FilesContentIndexer.name, () => {
start: jest.fn().mockRejectedValue(new Error('ocr init failed')),
stop: jest.fn().mockResolvedValue(undefined)
}
- jest.spyOn(PdfOCRWorkerManager, 'getInstance').mockReturnValue(ocrManager as any)
+ jest.spyOn(OCRManager, 'getInstance').mockReturnValue(ocrManager as any)
filesParser.allPaths.mockReturnValue(
asyncGen([[9, FILE_REPOSITORY.USER, [{ realPath: '/u/jane', pathPrefix: 'files/personal', isDir: true }]]] as any)
)
@@ -282,7 +282,7 @@ describe(FilesContentIndexer.name, () => {
})
it('should parse content and return null when parser returns empty or throws', async () => {
- ;(service as any).pdfOcrWorkerManager = { worker: { id: 'worker-1' } }
+ ;(service as any).ocrManager = { worker: { id: 'worker-1' } }
const docTextifySpy = jest.spyOn(docTextifyModule, 'docTextify')
docTextifySpy.mockResolvedValueOnce('')
docTextifySpy.mockRejectedValueOnce(new Error('parse failed'))
diff --git a/backend/src/applications/files/services/files-content-indexer.service.ts b/backend/src/applications/files/services/files-content-indexer.service.ts
index 1f3e6622..d7863b44 100644
--- a/backend/src/applications/files/services/files-content-indexer.service.ts
+++ b/backend/src/applications/files/services/files-content-indexer.service.ts
@@ -7,7 +7,7 @@ import { FileIndexContext, FileParseContext } from '../interfaces/file-parse-ind
import { FilesContentStore } from '../models/files-content-store'
import { FileContent } from '../schemas/file-content.interface'
import { docTextify } from '../utils/doc-textify/doc-textify'
-import { PdfOCRWorkerManager } from '../utils/doc-textify/utils/pdf-ocr'
+import { OCRManager } from '../utils/doc-textify/utils/ocr'
import { getExtensionWithoutDot, getMimeType } from '../utils/files'
import { FilesParser } from './files-parser.service'
import { genIndexingKey } from '../utils/indexing'
@@ -18,7 +18,7 @@ import { Cache } from '../../../infrastructure/cache/services/cache.service'
export class FilesContentIndexer {
private readonly maxDocumentSize = 150 * 1_000_000
private readonly logger = new Logger(FilesContentIndexer.name)
- private pdfOcrWorkerManager: PdfOCRWorkerManager | null = null
+ private ocrManager: OCRManager | null = null
constructor(
private readonly cache: Cache,
@@ -62,9 +62,9 @@ export class FilesContentIndexer {
}
async parseAndIndexAllFiles(userIds?: number[], spaceIds?: number[], shareIds?: number[]): Promise<void> {
- this.pdfOcrWorkerManager = PdfOCRWorkerManager.getInstance(this.logger)
+ this.ocrManager = OCRManager.getInstance(this.logger)
try {
- await this.pdfOcrWorkerManager.start()
+ await this.ocrManager.start()
} catch (e) {
this.logger.warn({ tag: this.parseAndIndexAllFiles.name, msg: `unable to initialize OCR worker: ${e}` })
}
@@ -84,8 +84,8 @@ export class FilesContentIndexer {
await this.filesIndexer.cleanIndexes(indexSuffixes)
}
} finally {
- await this.pdfOcrWorkerManager?.stop()
- this.pdfOcrWorkerManager = null
+ await this.ocrManager?.stop()
+ this.ocrManager = null
}
}
@@ -147,7 +147,7 @@ export class FilesContentIndexer {
} else {
this.logger.log({
tag: this.indexFiles.name,
- msg: `${indexSuffix} - indexed: ${indexedRecords - errorRecords}, errors: ${errorRecords}, deleted: ${recordsToDelete.length}`
+ msg: `${indexSuffix} - indexed: ${indexedRecords - errorRecords}, deleted: ${recordsToDelete.length}, errors: ${errorRecords}`
})
}
}
@@ -224,7 +224,7 @@ export class FilesContentIndexer {
{
newlineDelimiter: ' ',
minCharsToExtract: 10,
- ocrWorker: this.pdfOcrWorkerManager?.worker
+ ocrWorker: this.ocrManager?.worker
},
{
extension: extension,
diff --git a/backend/src/applications/files/services/files-event-manager.service.ts b/backend/src/applications/files/services/files-event-manager.service.ts
index ec2c7c47..6ca7734a 100644
--- a/backend/src/applications/files/services/files-event-manager.service.ts
+++ b/backend/src/applications/files/services/files-event-manager.service.ts
@@ -37,12 +37,12 @@ export class FilesEventManager implements OnModuleDestroy {
await this.flushEvents()
}
- private readonly onFileEvent = (fileEvent: FileEventType): void => {
+ private readonly onFileEvent = (fEvent: FileEventType): void => {
this.logger.verbose({
tag: this.onFileEvent.name,
- msg: `Receiving: user:${fileEvent.user.login} action:${fileEvent.action} on ${fileEvent.space.alias}`
+ msg: `Receiving: user:${fEvent.user.login} action:${fEvent.action} url:${fEvent.space.url}`
})
- this.eventsBuffer.push(fileEvent)
+ this.eventsBuffer.push(fEvent)
if (this.eventsBuffer.length >= this.MAX_BUFFER_SIZE) {
void this.flushEvents()
return
diff --git a/backend/src/applications/files/utils/doc-textify/utils/pdf-ocr.ts b/backend/src/applications/files/utils/doc-textify/utils/ocr.ts
similarity index 63%
rename from backend/src/applications/files/utils/doc-textify/utils/pdf-ocr.ts
rename to backend/src/applications/files/utils/doc-textify/utils/ocr.ts
index 69a23040..bc204bb3 100644
--- a/backend/src/applications/files/utils/doc-textify/utils/pdf-ocr.ts
+++ b/backend/src/applications/files/utils/doc-textify/utils/ocr.ts
@@ -3,19 +3,21 @@ import path from 'node:path'
import Tesseract from 'tesseract.js'
import { configuration } from '../../../../../configuration/config.environment'
import { makeDir } from '../../files'
+import type { FilesContentIndexingOCRConfig } from '../../../files.config'
-export type PdfOCRWorker = Awaited<ReturnType<typeof Tesseract.createWorker>>
-type PdfOCRWorkerOptions = NonNullable<Parameters<typeof Tesseract.createWorker>[2]>
+type OCRWorker = Awaited<ReturnType<typeof Tesseract.createWorker>>
+type OCRWorkerOptions = NonNullable<Parameters<typeof Tesseract.createWorker>[2]>
-export class PdfOCRWorkerManager {
- private static instance: PdfOCRWorkerManager
- public worker: PdfOCRWorker | null
- private readonly ocrLanguagesPath = path.resolve(__dirname, '../../../assets/ocr-languages')
+export class OCRManager {
+ private static instance: OCRManager
+ public worker: OCRWorker | null
+ private readonly ocrConfig: FilesContentIndexingOCRConfig = configuration.applications.files.contentIndexing.ocr
+ private readonly ocrLanguagesPath = this.ocrConfig.languagesPath ?? path.resolve(__dirname, '../../../assets/ocr-languages')
private readonly ocrUserDefinedDpi = '300'
private readonly ocrTrainedDataExtension = '.traineddata'
private readonly ocrTrainedDataGzipExtension = '.traineddata.gz'
private logger: Logger
- private workerInitializationPromise: Promise<PdfOCRWorker | null> | null
+ private workerInitializationPromise: Promise<OCRWorker | null> | null
private constructor(logger: Logger) {
this.logger = logger
@@ -23,24 +25,25 @@ export class PdfOCRWorkerManager {
this.workerInitializationPromise = null
}
- static getInstance(logger: Logger): PdfOCRWorkerManager {
- if (!PdfOCRWorkerManager.instance) {
- PdfOCRWorkerManager.instance = new PdfOCRWorkerManager(logger)
+ static getInstance(logger: Logger): OCRManager {
+ if (!OCRManager.instance) {
+ OCRManager.instance = new OCRManager(logger)
}
- return PdfOCRWorkerManager.instance
+ return OCRManager.instance
}
- async start(): Promise<PdfOCRWorker | null> {
+ async start(): Promise<OCRWorker | null> {
if (this.worker) {
return this.worker
}
if (this.workerInitializationPromise) {
return this.workerInitializationPromise
}
- this.logger.verbose({ tag: this.constructor.name, msg: 'Starting OCR' })
+
this.workerInitializationPromise = this.createWorkerFromConfiguration()
.then((worker) => {
this.worker = worker
+ if (this.worker !== null) this.logger.verbose({ tag: this.constructor.name, msg: 'Started' })
return worker
})
.finally(() => {
@@ -50,7 +53,6 @@ export class PdfOCRWorkerManager {
}
async stop(): Promise<void> {
- this.logger.verbose({ tag: this.constructor.name, msg: 'Stopping OCR' })
if (this.workerInitializationPromise) {
await this.workerInitializationPromise.catch((e) =>
this.logger.error({
@@ -64,15 +66,19 @@ export class PdfOCRWorkerManager {
if (!worker) {
return
}
- await worker.terminate().catch((e) =>
- this.logger.error({
- tag: this.constructor.name,
- msg: `${e}`
- })
- )
+
+ await worker
+ .terminate()
+ .then(() => this.logger.verbose({ tag: this.constructor.name, msg: 'Stopped' }))
+ .catch((e) =>
+ this.logger.error({
+ tag: this.constructor.name,
+ msg: `${e}`
+ })
+ )
}
- private async createConfiguredWorker(languages: string[], options: PdfOCRWorkerOptions): Promise<PdfOCRWorker> {
+ private async createConfiguredWorker(languages: string[], options: OCRWorkerOptions): Promise<OCRWorker> {
const worker = await Tesseract.createWorker(languages, Tesseract.OEM.LSTM_ONLY, {
...options,
errorHandler: (e: unknown) =>
@@ -90,9 +96,8 @@ export class PdfOCRWorkerManager {
return worker
}
- private async createWorkerFromConfiguration(): Promise<PdfOCRWorker | null> {
- const ocrOptions = configuration.applications.files.contentIndexing.ocr
- if (!ocrOptions.enabled) {
+ private async createWorkerFromConfiguration(): Promise<OCRWorker | null> {
+ if (!this.ocrConfig.enabled) {
return null
}
try {
@@ -103,14 +108,14 @@ export class PdfOCRWorkerManager {
msg: `unable to create languages directory: ${e}`
})
}
- if (ocrOptions.offline) {
+ if (this.ocrConfig.offline) {
const offlineWorkerOptions = {
langPath: this.ocrLanguagesPath,
cacheMethod: 'none' as const
}
try {
// First try for gzipped files: <lang>.traineddata.gz
- return await this.createConfiguredWorker(ocrOptions.languages, {
+ return await this.createConfiguredWorker(this.ocrConfig.languages, {
...offlineWorkerOptions,
gzip: true
})
@@ -120,13 +125,13 @@ export class PdfOCRWorkerManager {
msg: `unable to load offline OCR languages as ${this.ocrTrainedDataGzipExtension}, retrying with ${this.ocrTrainedDataExtension}: ${error}`
})
// Fallback for non-gz files: <lang>.traineddata
- return this.createConfiguredWorker(ocrOptions.languages, {
+ return this.createConfiguredWorker(this.ocrConfig.languages, {
...offlineWorkerOptions,
gzip: false
})
}
}
- return this.createConfiguredWorker(ocrOptions.languages, {
+ return this.createConfiguredWorker(this.ocrConfig.languages, {
cachePath: this.ocrLanguagesPath,
cacheMethod: 'write' as const
})
diff --git a/environment/environment.dist.yaml b/environment/environment.dist.yaml
index 78a4ad6b..9e76cf28 100755
--- a/environment/environment.dist.yaml
+++ b/environment/environment.dist.yaml
@@ -131,6 +131,9 @@ auth:
# ca: [/app/certs/ca.pem]
# optional
tlsOptions:
+ # baseDN: Distinguished name (e.g.: ou=people,dc=ldap,dc=sync-in,dc=com)
+ # Used as the search base for users, and for groups when adminGroup is a CN.
+ # required
baseDN: ou=people,dc=ldap,dc=sync-in,dc=com
# filter, e.g: (acl=admin)
# Appended as-is to the LDAP search filter (trusted config).
@@ -314,6 +317,10 @@ applications:
# To download languages, use this scheme: https://cdn.jsdelivr.net/npm/@tesseract.js-data/<lang>@1.0.0/4.0.0_best_int/<lang>.traineddata.gz
# default: false
offline: false
+ # Path to local OCR language files
+ # Used when offline mode is enabled or to override default location
+ # default: built-in OCR directory
+ languagesPath:
# Show files starting with a dot in the file explorer
# default: false
showHiddenFiles: false