Skip to content

Build your personal knowledge base with Trilium Notes

17.4 KB
Raw Blame
import { getTesseractCode } from '@triliumnext/commons';
import { becca, blob as blobService, entity_changes as entityChangesService, getLog, options } from '@triliumnext/core';

import { asBuffer } from '../binary.js';
import sql from '../sql.js';
import { FileProcessor } from './processors/file_processor.js';
import { ImageProcessor } from './processors/image_processor.js';
import { OfficeProcessor } from './processors/office_processor.js';
import { PDFProcessor } from './processors/pdf_processor.js';

export interface OCRResult {
    text: string;
    confidence: number;
    extractedAt: string;
    language?: string;
    pageCount?: number;
    /** Pages whose OCR threw, for a document whose other pages still produced a result. */
    failedPageCount?: number;
    processingType?: string;
}

export interface OCRProcessingOptions {
    language?: string;
    forceReprocess?: boolean;
    confidence?: number;
    enablePDFTextExtraction?: boolean;
    mimeType?: string;
}

/**
 * OCR Service for extracting text from images and other OCR-able objects
 * Uses Tesseract.js for text recognition
 */
class OCRService {
    private processors: Map<string, FileProcessor> = new Map();

    constructor() {
        this.processors.set('image', new ImageProcessor());
        this.processors.set('pdf', new PDFProcessor());
        this.processors.set('office', new OfficeProcessor());
    }

    /**
     * Resolves the Tesseract language code(s) for OCR processing.
     *
     * Priority:
     * 1. Explicitly passed `language` option (e.g. from API call)
     * 2. The note's `language` label (mapped via {@link getTesseractCode})
     * 3. All enabled content languages joined with `+`
     * 4. The UI locale
     * 5. Fallback to `eng`
     */
    resolveOcrLanguage(noteId?: string, explicitLanguage?: string): string {
        // 1. Explicit language from caller
        if (explicitLanguage) {
            return explicitLanguage;
        }

        // 2. Note's language label
        if (noteId) {
            const note = becca.getNote(noteId);
            const noteLanguage = note?.getLabelValue("language");
            if (noteLanguage) {
                const code = getTesseractCode(noteLanguage);
                if (code) {
                    return code;
                }
            }
        }

        // 3. All enabled content languages
        try {
            const languagesJson = options.getOption("languages");
            const enabledLanguages = JSON.parse(languagesJson || "[]") as string[];
            if (enabledLanguages.length > 0) {
                const codes = enabledLanguages
                    .map((id) => getTesseractCode(id))
                    .filter((code): code is string => code !== null);
                // Deduplicate (e.g. en + en-GB both map to eng)
                const unique = [...new Set(codes)];
                if (unique.length > 0) {
                    return unique.join("+");
                }
            }
        } catch {
            // Fall through
        }

        // 4. UI locale
        try {
            const uiLocale = options.getOption("locale");
            if (uiLocale) {
                const code = getTesseractCode(uiLocale);
                if (code) {
                    return code;
                }
            }
        } catch {
            // Fall through
        }

        // 5. Fallback
        return "eng";
    }


    /**
     * Extract text from file buffer using appropriate processor
     */
    async extractTextFromFile(fileBuffer: Buffer, mimeType: string, options: OCRProcessingOptions = {}): Promise<OCRResult> {
        getLog().info(`Starting OCR text extraction for MIME type: ${mimeType} with language: ${options.language || "eng"}`);

        const processor = this.getProcessorForMimeType(mimeType);
        if (!processor) {
            throw new Error(`No processor found for MIME type: ${mimeType}`);
        }

        const result = await processor.extractText(fileBuffer, { ...options, mimeType });
        result.processingType = processor.getProcessingType();

        getLog().info(`OCR extraction completed. Confidence: ${Math.round(result.confidence * 100)}%, Text length: ${result.text.length}`);
        return result;
    }

    /**
     * Process OCR for a note (image type)
     */
    async processNoteOCR(noteId: string, options: OCRProcessingOptions = {}): Promise<OCRResult | null> {
        const note = becca.getNote(noteId);
        if (!note) {
            getLog().error(`Note ${noteId} not found`);
            return null;
        }

        return this.processEntityOCR({
            entityId: noteId,
            entityType: 'note',
            category: note.type,
            mime: note.mime,
            blobId: note.blobId,
            isProtected: !!note.isProtected,
            isContentAvailable: note.isContentAvailable(),
            languageNoteId: noteId,
            getContent: () => note.getContent()
        }, options);
    }

    /**
     * Process OCR for an attachment
     */
    async processAttachmentOCR(attachmentId: string, options: OCRProcessingOptions = {}): Promise<OCRResult | null> {
        const attachment = becca.getAttachment(attachmentId);
        if (!attachment) {
            getLog().error(`Attachment ${attachmentId} not found`);
            return null;
        }

        return this.processEntityOCR({
            entityId: attachmentId,
            entityType: 'attachment',
            category: attachment.role,
            mime: attachment.mime,
            blobId: attachment.blobId,
            isProtected: !!attachment.isProtected,
            isContentAvailable: attachment.isContentAvailable(),
            languageNoteId: attachment.ownerId,
            getContent: () => attachment.getContent()
        }, options);
    }

    /**
     * Shared OCR processing logic for both notes and attachments.
     */
    private async processEntityOCR(entity: {
        entityId: string;
        entityType: string;
        category: string;
        mime: string;
        blobId: string | undefined;
        isProtected: boolean;
        isContentAvailable: boolean;
        languageNoteId: string;
        getContent: () => string | Uint8Array;
    }, options: OCRProcessingOptions = {}): Promise<OCRResult | null> {
        const { entityId, entityType, category, mime, blobId, isProtected, isContentAvailable, languageNoteId } = entity;

        if (!['image', 'file'].includes(category)) {
            getLog().info(`${entityType} ${entityId} is not an image or file, skipping OCR`);
            return null;
        }

        if (!this.getProcessorForMimeType(mime)) {
            getLog().info(`${entityType} ${entityId} has unsupported MIME type ${mime} for text extraction, skipping`);
            return null;
        }

        if (!options.forceReprocess && this.hasStoredOCRResult(blobId)) {
            getLog().info(`OCR already exists for ${entityType} ${entityId}, skipping`);
            return null;
        }

        // Without the key the content is not readable, so there is nothing to recognise. This is an
        // ordinary skip rather than a failure: the entity is picked up again once the vault is open.
        if (!isContentAvailable) {
            getLog().info(`${entityType} ${entityId} is protected and no protected session is available, skipping OCR`);
            return null;
        }

        try {
            const content = toBinaryContent(entity.getContent());
            if (!content) {
                throw new Error(`Cannot get content for ${entityType} ${entityId}`);
            }

            const language = this.resolveOcrLanguage(languageNoteId, options.language);
            const ocrResult = await this.extractTextFromFile(content, mime, { ...options, language });

            this.storeOCRResult(blobId, ocrResult, isProtected);

            return ocrResult;
        } catch (error) {
            getLog().error(`Failed to process OCR for ${entityType} ${entityId}: ${error}`);
            throw error;
        }
    }

    /**
     * Store OCR result in blob.
     *
     * The text is encrypted for a protected entity, since it is a readable rendering of content that
     * is itself only stored encrypted.
     */
    storeOCRResult(blobId: string | undefined, ocrResult: OCRResult, isProtected: boolean): void {
        if (!blobId) {
            getLog().error('Cannot store OCR result: blobId is undefined');
            return;
        }

        try {
            const textRepresentation = blobService.encryptTextRepresentation(ocrResult.text, isProtected);

            sql.execute(`
                UPDATE blobs SET textRepresentation = ?
                WHERE blobId = ?
            `, [textRepresentation, blobId]);

            entityChangesService.putBlobEntityChange(blobId);

            getLog().info(`Stored OCR result for blob ${blobId}`);
        } catch (error) {
            getLog().error(`Failed to store OCR result for blob ${blobId}: ${error}`);
            throw error;
        }
    }

    /**
     * Check whether a blob already has a stored text representation.
     */
    private hasStoredOCRResult(blobId: string | undefined): boolean {
        if (!blobId) {
            return false;
        }

        const row = sql.getRow<{ textRepresentation: string | null }>(
            `SELECT textRepresentation FROM blobs WHERE blobId = ?`,
            [blobId]
        );

        return !!row?.textRepresentation;
    }

    // Batch processing state
    private batchProcessingState: {
        inProgress: boolean;
        total: number;
        processed: number;
        failed: number;
        startTime?: Date;
    } = {
        inProgress: false,
        total: 0,
        processed: 0,
        failed: 0
    };

    /**
     * Start batch OCR processing with progress tracking
     */
    async startBatchProcessing(): Promise<{ success: boolean; message?: string }> {
        if (this.batchProcessingState.inProgress) {
            return { success: false, message: 'Batch processing already in progress' };
        }

        try {
            const blobsNeedingOCR = this.getBlobsNeedingOCR();

            if (blobsNeedingOCR.length === 0) {
                return { success: false, message: 'No images found that need OCR processing' };
            }

            this.batchProcessingState = {
                inProgress: true,
                total: blobsNeedingOCR.length,
                processed: 0,
                failed: 0,
                startTime: new Date()
            };

            // Start processing in background
            this.processBlobs(blobsNeedingOCR).catch(error => {
                getLog().error(`Batch processing failed: ${error instanceof Error ? error.message : String(error)}`);
            }).finally(() => {
                this.batchProcessingState.inProgress = false;
            });

            return { success: true };
        } catch (error) {
            getLog().error(`Failed to start batch processing: ${error instanceof Error ? error.message : String(error)}`);
            return { success: false, message: error instanceof Error ? error.message : String(error) };
        }
    }

    /**
     * Get batch processing progress
     */
    getBatchProgress(): { inProgress: boolean; total: number; processed: number; failed: number; percentage?: number; startTime?: Date } {
        const result: { inProgress: boolean; total: number; processed: number; failed: number; percentage?: number; startTime?: Date } = { ...this.batchProcessingState };
        if (result.total > 0) {
            result.percentage = (result.processed / result.total) * 100;
        }
        return result;
    }

    /**
     * Cancel batch processing
     */
    cancelBatchProcessing(): void {
        if (this.batchProcessingState.inProgress) {
            this.batchProcessingState.inProgress = false;
            getLog().info('Batch OCR processing cancelled');
        }
    }

    /**
     * Process a list of blobs sequentially, updating batch progress.
     */
    private async processBlobs(blobs: Array<{ entityType: 'note' | 'attachment'; entityId: string }>): Promise<void> {
        getLog().info(`Starting batch OCR processing of ${blobs.length} items...`);

        for (const blob of blobs) {
            if (!this.batchProcessingState.inProgress) {
                break;
            }

            try {
                await this.processOcrEntity(blob);
            } catch {
                // Tolerate broken or undecodable files: count the failure and move on.
                // The failure itself is already logged by processOcrEntity.
                this.batchProcessingState.failed++;
            }

            this.batchProcessingState.processed++;

            // Small delay to keep the system responsive; recognition itself already runs
            // in a worker thread, so this only needs to yield, not throttle.
            await new Promise(resolve => setTimeout(resolve, 50));
        }

        getLog().info(`Batch OCR processing completed. Processed ${this.batchProcessingState.processed} files (${this.batchProcessingState.failed} failed).`);
    }

    /**
     * Process OCR for a single entity (note or attachment) by type.
     */
    private async processOcrEntity(entity: { entityType: 'note' | 'attachment'; entityId: string }): Promise<void> {
        if (entity.entityType === 'note') {
            await this.processNoteOCR(entity.entityId);
        } else {
            await this.processAttachmentOCR(entity.entityId);
        }
    }

    /**
     * Get processor for a given MIME type
     */
    private getProcessorForMimeType(mimeType: string): FileProcessor | null {
        for (const processor of this.processors.values()) {
            if (processor.canProcess(mimeType)) {
                return processor;
            }
        }
        return null;
    }

    /**
     * Get all MIME types supported by all registered processors
     */
    getAllSupportedMimeTypes(): string[] {
        const supportedTypes = new Set<string>();

        // Gather MIME types from all registered processors
        for (const processor of this.processors.values()) {
            const processorTypes = processor.getSupportedMimeTypes();
            processorTypes.forEach(type => supportedTypes.add(type));
        }

        return Array.from(supportedTypes);
    }


    /**
     * Get blobs that need OCR processing (those without a stored text representation).
     * A NULL text representation means "never processed"; an empty string means
     * "processed, found no text" and is treated as done, so a blob is never re-OCR'd
     * on every batch run. Existing scanned PDFs stored before the OCR fallback can be
     * re-run individually via the manual "Process OCR" action (which forces reprocess).
     */
    getBlobsNeedingOCR(): Array<{ blobId: string; mimeType: string; entityType: 'note' | 'attachment'; entityId: string }> {
        try {
            const supportedMimes = this.getAllSupportedMimeTypes();
            const placeholders = supportedMimes.map(() => '?').join(', ');

            const noteBlobs = sql.getRows<{
                blobId: string;
                mimeType: string;
                entityId: string;
            }>(`
                SELECT n.blobId, n.mime as mimeType, n.noteId as entityId
                FROM notes n
                JOIN blobs b ON n.blobId = b.blobId
                WHERE (n.type = 'image' OR (n.type = 'file' AND n.mime IN (${placeholders})))
                AND n.isDeleted = 0
                AND n.blobId IS NOT NULL
                AND b.textRepresentation IS NULL
            `, supportedMimes);

            const attachmentBlobs = sql.getRows<{
                blobId: string;
                mimeType: string;
                entityId: string;
            }>(`
                SELECT a.blobId, a.mime as mimeType, a.attachmentId as entityId
                FROM attachments a
                JOIN blobs b ON a.blobId = b.blobId
                WHERE (a.role = 'image' OR (a.role = 'file' AND a.mime IN (${placeholders})))
                AND a.isDeleted = 0
                AND a.blobId IS NOT NULL
                AND b.textRepresentation IS NULL
            `, supportedMimes);

            // Combine results
            const result = [
                ...noteBlobs.map(blob => ({ ...blob, entityType: 'note' as const })),
                ...attachmentBlobs.map(blob => ({ ...blob, entityType: 'attachment' as const }))
            ];

            // Return all results (no need to filter by MIME type as we already did in the query)
            return result;
        } catch (error) {
            getLog().error(`Failed to get blobs needing OCR: ${error}`);
            return [];
        }
    }

}

export default new OCRService();

/**
 * Narrows an entity's content to the `Buffer` the processors take, or null when there are no bytes to
 * recognise — a string has no binary form, and neither does an empty buffer.
 *
 * The conversion matters because the two kinds of content arrive differently typed. Unprotected
 * content is already a `Buffer`: better-sqlite3 returns one for a BLOB column and nothing rewraps it
 * on the way through Becca. Protected content is decrypted on the way out, and the decryption builds
 * its result with `new Uint8Array(...)`, which is not a `Buffer` — so testing for one rejected every
 * protected image as unreadable. Same bytes either way, so wrap rather than reject.
 */
function toBinaryContent(content: string | Uint8Array): Buffer | null {
    if (typeof content === "string" || content.byteLength === 0) {
        return null;
    }

    return asBuffer(content);
}