Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
import { getTesseractCode } from '@triliumnext/commons';
import { becca, blob as blobService, entity_changes as entityChangesService, getLog, options } from '@triliumnext/core';
import { asBuffer } from '../binary.js';
import sql from '../sql.js';
import { FileProcessor } from './processors/file_processor.js';
import { ImageProcessor } from './processors/image_processor.js';
import { OfficeProcessor } from './processors/office_processor.js';
import { PDFProcessor } from './processors/pdf_processor.js';
export interface OCRResult {
text: string;
confidence: number;
extractedAt: string;
language?: string;
pageCount?: number;
/** Pages whose OCR threw, for a document whose other pages still produced a result. */
failedPageCount?: number;
processingType?: string;
}
export interface OCRProcessingOptions {
language?: string;
forceReprocess?: boolean;
confidence?: number;
enablePDFTextExtraction?: boolean;
mimeType?: string;
}
/**
* OCR Service for extracting text from images and other OCR-able objects
* Uses Tesseract.js for text recognition
*/
class OCRService {
private processors: Map<string, FileProcessor> = new Map();
constructor() {
this.processors.set('image', new ImageProcessor());
this.processors.set('pdf', new PDFProcessor());
this.processors.set('office', new OfficeProcessor());
}
/**
* Resolves the Tesseract language code(s) for OCR processing.
*
* Priority:
* 1. Explicitly passed `language` option (e.g. from API call)
* 2. The note's `language` label (mapped via {@link getTesseractCode})
* 3. All enabled content languages joined with `+`
* 4. The UI locale
* 5. Fallback to `eng`
*/
resolveOcrLanguage(noteId?: string, explicitLanguage?: string): string {
// 1. Explicit language from caller
if (explicitLanguage) {
return explicitLanguage;
}
// 2. Note's language label
if (noteId) {
const note = becca.getNote(noteId);
const noteLanguage = note?.getLabelValue("language");
if (noteLanguage) {
const code = getTesseractCode(noteLanguage);
if (code) {
return code;
}
}
}
// 3. All enabled content languages
try {
const languagesJson = options.getOption("languages");
const enabledLanguages = JSON.parse(languagesJson || "[]") as string[];
if (enabledLanguages.length > 0) {
const codes = enabledLanguages
.map((id) => getTesseractCode(id))
.filter((code): code is string => code !== null);
// Deduplicate (e.g. en + en-GB both map to eng)
const unique = [...new Set(codes)];
if (unique.length > 0) {
return unique.join("+");
}
}
} catch {
// Fall through
}
// 4. UI locale
try {
const uiLocale = options.getOption("locale");
if (uiLocale) {
const code = getTesseractCode(uiLocale);
if (code) {
return code;
}
}
} catch {
// Fall through
}
// 5. Fallback
return "eng";
}
/**
* Extract text from file buffer using appropriate processor
*/
async extractTextFromFile(fileBuffer: Buffer, mimeType: string, options: OCRProcessingOptions = {}): Promise<OCRResult> {
getLog().info(`Starting OCR text extraction for MIME type: ${mimeType} with language: ${options.language || "eng"}`);
const processor = this.getProcessorForMimeType(mimeType);
if (!processor) {
throw new Error(`No processor found for MIME type: ${mimeType}`);
}
const result = await processor.extractText(fileBuffer, { ...options, mimeType });
result.processingType = processor.getProcessingType();
getLog().info(`OCR extraction completed. Confidence: ${Math.round(result.confidence * 100)}%, Text length: ${result.text.length}`);
return result;
}
/**
* Process OCR for a note (image type)
*/
async processNoteOCR(noteId: string, options: OCRProcessingOptions = {}): Promise<OCRResult | null> {
const note = becca.getNote(noteId);
if (!note) {
getLog().error(`Note ${noteId} not found`);
return null;
}
return this.processEntityOCR({
entityId: noteId,
entityType: 'note',
category: note.type,
mime: note.mime,
blobId: note.blobId,
isProtected: !!note.isProtected,
isContentAvailable: note.isContentAvailable(),
languageNoteId: noteId,
getContent: () => note.getContent()
}, options);
}
/**
* Process OCR for an attachment
*/
async processAttachmentOCR(attachmentId: string, options: OCRProcessingOptions = {}): Promise<OCRResult | null> {
const attachment = becca.getAttachment(attachmentId);
if (!attachment) {
getLog().error(`Attachment ${attachmentId} not found`);
return null;
}
return this.processEntityOCR({
entityId: attachmentId,
entityType: 'attachment',
category: attachment.role,
mime: attachment.mime,
blobId: attachment.blobId,
isProtected: !!attachment.isProtected,
isContentAvailable: attachment.isContentAvailable(),
languageNoteId: attachment.ownerId,
getContent: () => attachment.getContent()
}, options);
}
/**
* Shared OCR processing logic for both notes and attachments.
*/
private async processEntityOCR(entity: {
entityId: string;
entityType: string;
category: string;
mime: string;
blobId: string | undefined;
isProtected: boolean;
isContentAvailable: boolean;
languageNoteId: string;
getContent: () => string | Uint8Array;
}, options: OCRProcessingOptions = {}): Promise<OCRResult | null> {
const { entityId, entityType, category, mime, blobId, isProtected, isContentAvailable, languageNoteId } = entity;
if (!['image', 'file'].includes(category)) {
getLog().info(`${entityType} ${entityId} is not an image or file, skipping OCR`);
return null;
}
if (!this.getProcessorForMimeType(mime)) {
getLog().info(`${entityType} ${entityId} has unsupported MIME type ${mime} for text extraction, skipping`);
return null;
}
if (!options.forceReprocess && this.hasStoredOCRResult(blobId)) {
getLog().info(`OCR already exists for ${entityType} ${entityId}, skipping`);
return null;
}
// Without the key the content is not readable, so there is nothing to recognise. This is an
// ordinary skip rather than a failure: the entity is picked up again once the vault is open.
if (!isContentAvailable) {
getLog().info(`${entityType} ${entityId} is protected and no protected session is available, skipping OCR`);
return null;
}
try {
const content = toBinaryContent(entity.getContent());
if (!content) {
throw new Error(`Cannot get content for ${entityType} ${entityId}`);
}
const language = this.resolveOcrLanguage(languageNoteId, options.language);
const ocrResult = await this.extractTextFromFile(content, mime, { ...options, language });
this.storeOCRResult(blobId, ocrResult, isProtected);
return ocrResult;
} catch (error) {
getLog().error(`Failed to process OCR for ${entityType} ${entityId}: ${error}`);
throw error;
}
}
/**
* Store OCR result in blob.
*
* The text is encrypted for a protected entity, since it is a readable rendering of content that
* is itself only stored encrypted.
*/
storeOCRResult(blobId: string | undefined, ocrResult: OCRResult, isProtected: boolean): void {
if (!blobId) {
getLog().error('Cannot store OCR result: blobId is undefined');
return;
}
try {
const textRepresentation = blobService.encryptTextRepresentation(ocrResult.text, isProtected);
sql.execute(`
UPDATE blobs SET textRepresentation = ?
WHERE blobId = ?
`, [textRepresentation, blobId]);
entityChangesService.putBlobEntityChange(blobId);
getLog().info(`Stored OCR result for blob ${blobId}`);
} catch (error) {
getLog().error(`Failed to store OCR result for blob ${blobId}: ${error}`);
throw error;
}
}
/**
* Check whether a blob already has a stored text representation.
*/
private hasStoredOCRResult(blobId: string | undefined): boolean {
if (!blobId) {
return false;
}
const row = sql.getRow<{ textRepresentation: string | null }>(
`SELECT textRepresentation FROM blobs WHERE blobId = ?`,
[blobId]
);
return !!row?.textRepresentation;
}
// Batch processing state
private batchProcessingState: {
inProgress: boolean;
total: number;
processed: number;
failed: number;
startTime?: Date;
} = {
inProgress: false,
total: 0,
processed: 0,
failed: 0
};
/**
* Start batch OCR processing with progress tracking
*/
async startBatchProcessing(): Promise<{ success: boolean; message?: string }> {
if (this.batchProcessingState.inProgress) {
return { success: false, message: 'Batch processing already in progress' };
}
try {
const blobsNeedingOCR = this.getBlobsNeedingOCR();
if (blobsNeedingOCR.length === 0) {
return { success: false, message: 'No images found that need OCR processing' };
}
this.batchProcessingState = {
inProgress: true,
total: blobsNeedingOCR.length,
processed: 0,
failed: 0,
startTime: new Date()
};
// Start processing in background
this.processBlobs(blobsNeedingOCR).catch(error => {
getLog().error(`Batch processing failed: ${error instanceof Error ? error.message : String(error)}`);
}).finally(() => {
this.batchProcessingState.inProgress = false;
});
return { success: true };
} catch (error) {
getLog().error(`Failed to start batch processing: ${error instanceof Error ? error.message : String(error)}`);
return { success: false, message: error instanceof Error ? error.message : String(error) };
}
}
/**
* Get batch processing progress
*/
getBatchProgress(): { inProgress: boolean; total: number; processed: number; failed: number; percentage?: number; startTime?: Date } {
const result: { inProgress: boolean; total: number; processed: number; failed: number; percentage?: number; startTime?: Date } = { ...this.batchProcessingState };
if (result.total > 0) {
result.percentage = (result.processed / result.total) * 100;
}
return result;
}
/**
* Cancel batch processing
*/
cancelBatchProcessing(): void {
if (this.batchProcessingState.inProgress) {
this.batchProcessingState.inProgress = false;
getLog().info('Batch OCR processing cancelled');
}
}
/**
* Process a list of blobs sequentially, updating batch progress.
*/
private async processBlobs(blobs: Array<{ entityType: 'note' | 'attachment'; entityId: string }>): Promise<void> {
getLog().info(`Starting batch OCR processing of ${blobs.length} items...`);
for (const blob of blobs) {
if (!this.batchProcessingState.inProgress) {
break;
}
try {
await this.processOcrEntity(blob);
} catch {
// Tolerate broken or undecodable files: count the failure and move on.
// The failure itself is already logged by processOcrEntity.
this.batchProcessingState.failed++;
}
this.batchProcessingState.processed++;
// Small delay to keep the system responsive; recognition itself already runs
// in a worker thread, so this only needs to yield, not throttle.
await new Promise(resolve => setTimeout(resolve, 50));
}
getLog().info(`Batch OCR processing completed. Processed ${this.batchProcessingState.processed} files (${this.batchProcessingState.failed} failed).`);
}
/**
* Process OCR for a single entity (note or attachment) by type.
*/
private async processOcrEntity(entity: { entityType: 'note' | 'attachment'; entityId: string }): Promise<void> {
if (entity.entityType === 'note') {
await this.processNoteOCR(entity.entityId);
} else {
await this.processAttachmentOCR(entity.entityId);
}
}
/**
* Get processor for a given MIME type
*/
private getProcessorForMimeType(mimeType: string): FileProcessor | null {
for (const processor of this.processors.values()) {
if (processor.canProcess(mimeType)) {
return processor;
}
}
return null;
}
/**
* Get all MIME types supported by all registered processors
*/
getAllSupportedMimeTypes(): string[] {
const supportedTypes = new Set<string>();
// Gather MIME types from all registered processors
for (const processor of this.processors.values()) {
const processorTypes = processor.getSupportedMimeTypes();
processorTypes.forEach(type => supportedTypes.add(type));
}
return Array.from(supportedTypes);
}
/**
* Get blobs that need OCR processing (those without a stored text representation).
* A NULL text representation means "never processed"; an empty string means
* "processed, found no text" and is treated as done, so a blob is never re-OCR'd
* on every batch run. Existing scanned PDFs stored before the OCR fallback can be
* re-run individually via the manual "Process OCR" action (which forces reprocess).
*/
getBlobsNeedingOCR(): Array<{ blobId: string; mimeType: string; entityType: 'note' | 'attachment'; entityId: string }> {
try {
const supportedMimes = this.getAllSupportedMimeTypes();
const placeholders = supportedMimes.map(() => '?').join(', ');
const noteBlobs = sql.getRows<{
blobId: string;
mimeType: string;
entityId: string;
}>(`
SELECT n.blobId, n.mime as mimeType, n.noteId as entityId
FROM notes n
JOIN blobs b ON n.blobId = b.blobId
WHERE (n.type = 'image' OR (n.type = 'file' AND n.mime IN (${placeholders})))
AND n.isDeleted = 0
AND n.blobId IS NOT NULL
AND b.textRepresentation IS NULL
`, supportedMimes);
const attachmentBlobs = sql.getRows<{
blobId: string;
mimeType: string;
entityId: string;
}>(`
SELECT a.blobId, a.mime as mimeType, a.attachmentId as entityId
FROM attachments a
JOIN blobs b ON a.blobId = b.blobId
WHERE (a.role = 'image' OR (a.role = 'file' AND a.mime IN (${placeholders})))
AND a.isDeleted = 0
AND a.blobId IS NOT NULL
AND b.textRepresentation IS NULL
`, supportedMimes);
// Combine results
const result = [
...noteBlobs.map(blob => ({ ...blob, entityType: 'note' as const })),
...attachmentBlobs.map(blob => ({ ...blob, entityType: 'attachment' as const }))
];
// Return all results (no need to filter by MIME type as we already did in the query)
return result;
} catch (error) {
getLog().error(`Failed to get blobs needing OCR: ${error}`);
return [];
}
}
}
export default new OCRService();
/**
* Narrows an entity's content to the `Buffer` the processors take, or null when there are no bytes to
* recognise — a string has no binary form, and neither does an empty buffer.
*
* The conversion matters because the two kinds of content arrive differently typed. Unprotected
* content is already a `Buffer`: better-sqlite3 returns one for a BLOB column and nothing rewraps it
* on the way through Becca. Protected content is decrypted on the way out, and the decryption builds
* its result with `new Uint8Array(...)`, which is not a `Buffer` — so testing for one rejected every
* protected image as unreadable. Same bytes either way, so wrap rather than reject.
*/
function toBinaryContent(content: string | Uint8Array): Buffer | null {
if (typeof content === "string" || content.byteLength === 0) {
return null;
}
return asBuffer(content);
}