Skip to content

armgit / Orkas

Public Imported from GitHub
0

Orkas is an open-source (MIT), local-first AI team for your desktop: a Commander turns your goal into to-dos and coordinates 16 built-in specialist agents plus Claude Code, Codex, OpenCode, OpenClaw and Hermes, with projects, scheduled automations and connectors. Free app for macOS & Windows; built-in models or your own keys.

Sync local OCR file tool

wyt4565f2148e66
diff --git a/src/main/features/ocr_runtime.ts b/src/main/features/ocr_runtime.ts
new file mode 100644
index 00000000..ea679b69
--- /dev/null
+++ b/src/main/features/ocr_runtime.ts
@@ -0,0 +1,608 @@
+/**
+ * Local OCR runtime for `ocr_file`.
+ *
+ * Default engine: RapidOCR + ONNXRuntime CPU. PDF pages are rendered inside
+ * the same isolated Python venv with pypdfium2, then passed to RapidOCR as
+ * images. Callers get OCR markdown or a stable E_OCR_* error, both with
+ * process information suitable for the model to reason about what happened.
+ */
+
+import * as crypto from 'node:crypto';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+
+import {
+ PYTHON_VENV_PIP_CACHE_DIR,
+ PYTHON_VENV_UV_CACHE_DIR,
+ pythonPackageVenvDir,
+ userFileCacheDir,
+} from '../paths';
+import { bundledRuntimeEnv } from '../util/bundled-runtime';
+import { createLogger } from '../logger';
+
+const execFileAsync = promisify(execFile);
+const log = createLogger('ocr-runtime');
+
+export const OCR_RUNTIME_KEY = `ocr-rapidocr-3.9.0-onnxruntime-1.27.0-pypdfium2-5.10.1-${process.platform}-${process.arch}`;
+const RAPIDOCR_VERSION = '3.9.0';
+const ONNXRUNTIME_VERSION = '1.27.0';
+const PYPDFIUM2_VERSION = '5.10.1';
+const CACHE_VERSION = 1;
+const OCR_TIMEOUT_MS = 5 * 60 * 1000;
+const INSTALL_TIMEOUT_MS = 10 * 60 * 1000;
+const VERIFY_TIMEOUT_MS = 2 * 60 * 1000;
+const OCR_PROGRESS_HEARTBEAT_MS = 30_000;
+
+export interface OcrFileInput {
+ userId: string;
+ absPath: string;
+ pages?: string;
+ signal?: AbortSignal;
+ onProgress?: (event: { phase: string; message: string; data?: Record<string, unknown> }) => void;
+}
+
+type ProgressFn = NonNullable<OcrFileInput['onProgress']>;
+
+export type OcrFileResult =
+ | {
+ ok: true;
+ content: string;
+ pages: number[];
+ cached: boolean;
+ engine: string;
+ processLog: string[];
+ }
+ | {
+ ok: false;
+ errorCode:
+ | 'E_OCR_RUNTIME_MISSING'
+ | 'E_OCR_INSTALL_FAILED'
+ | 'E_OCR_UNSUPPORTED_FILE'
+ | 'E_OCR_FAILED'
+ | 'E_BAD_INPUT';
+ message: string;
+ processLog: string[];
+ };
+
+interface RuntimeReady {
+ ok: true;
+ python: string;
+ venv: string;
+ installed: boolean;
+}
+
+interface RuntimeMissing {
+ ok: false;
+ errorCode: 'E_OCR_RUNTIME_MISSING' | 'E_OCR_INSTALL_FAILED';
+ message: string;
+}
+
+type RuntimeResult = RuntimeReady | RuntimeMissing;
+
+type RuntimeVerification =
+ | { ok: true }
+ | { ok: false; message: string };
+
+function venvDir(): string {
+ return pythonPackageVenvDir(OCR_RUNTIME_KEY);
+}
+
+function venvPython(venv = venvDir()): string {
+ return process.platform === 'win32'
+ ? path.join(venv, 'Scripts', 'python.exe')
+ : path.join(venv, 'bin', 'python');
+}
+
+function isFile(p: string): boolean {
+ try { return fs.statSync(p).isFile(); }
+ catch { return false; }
+}
+
+function runtimeEnv(): NodeJS.ProcessEnv {
+ return {
+ ...process.env,
+ ...bundledRuntimeEnv(),
+ PYTHONNOUSERSITE: '1',
+ UV_CACHE_DIR: PYTHON_VENV_UV_CACHE_DIR,
+ PIP_CACHE_DIR: PYTHON_VENV_PIP_CACHE_DIR,
+ };
+}
+
+async function withHeartbeat<T>(
+ promise: Promise<T>,
+ onProgress: ProgressFn | undefined,
+ phase: string,
+ heartbeatMessage: string,
+): Promise<T> {
+ let tick = 0;
+ const timer = onProgress
+ ? setInterval(() => {
+ tick += 1;
+ onProgress({
+ phase,
+ message: heartbeatMessage,
+ data: { heartbeat: true, seconds: tick * (OCR_PROGRESS_HEARTBEAT_MS / 1000) },
+ });
+ }, OCR_PROGRESS_HEARTBEAT_MS)
+ : null;
+ try {
+ return await promise;
+ } finally {
+ if (timer) clearInterval(timer);
+ }
+}
+
+const VERIFY_RUNTIME_SCRIPT = String.raw`
+import pathlib
+import pypdfium2
+import onnxruntime
+import rapidocr as rapidocr_pkg
+from rapidocr import RapidOCR
+
+model_dir = pathlib.Path(rapidocr_pkg.__file__).resolve().parent / "models"
+RapidOCR(params={"Global.model_root_dir": str(model_dir)})
+print("ok")
+`;
+
+async function verifyRuntime(python: string, onProgress?: ProgressFn): Promise<RuntimeVerification> {
+ if (!isFile(python)) return { ok: false, message: `Python executable is missing: ${python}` };
+ try {
+ await withHeartbeat(
+ execFileAsync(
+ python,
+ ['-c', VERIFY_RUNTIME_SCRIPT],
+ { timeout: VERIFY_TIMEOUT_MS, env: runtimeEnv(), windowsHide: true, maxBuffer: 2 * 1024 * 1024 },
+ ),
+ onProgress,
+ 'ocr_runtime_verify',
+ 'Still checking local OCR runtime',
+ );
+ return { ok: true };
+ } catch (err) {
+ return { ok: false, message: (err as Error).message };
+ }
+}
+
+async function ensureRuntime(onProgress?: ProgressFn): Promise<RuntimeResult> {
+ const venv = venvDir();
+ const python = venvPython(venv);
+ onProgress?.({ phase: 'ocr_runtime_check', message: 'Checking local OCR runtime' });
+ const existingVerification = await verifyRuntime(python, onProgress);
+ if (existingVerification.ok === true) {
+ onProgress?.({ phase: 'ocr_runtime_ready', message: 'Local OCR runtime is ready' });
+ return { ok: true, python, venv, installed: false };
+ }
+
+ const env = runtimeEnv();
+ const uv = env.ORKAS_UV;
+ const bundledPython = env.ORKAS_PYTHON;
+ if (!uv || !bundledPython) {
+ onProgress?.({ phase: 'ocr_runtime_missing', message: 'Bundled Python/uv runtime is unavailable' });
+ return {
+ ok: false,
+ errorCode: 'E_OCR_RUNTIME_MISSING',
+ message: 'Bundled Python/uv runtime is unavailable, so Local OCR cannot be installed on this build.',
+ };
+ }
+
+ try {
+ fs.mkdirSync(path.dirname(venv), { recursive: true });
+ onProgress?.({
+ phase: 'ocr_runtime_install',
+ message: 'Creating local OCR Python environment',
+ data: { runtimePath: venv },
+ });
+ await withHeartbeat(
+ execFileAsync(uv, ['venv', '--python', bundledPython, venv], {
+ timeout: INSTALL_TIMEOUT_MS,
+ env,
+ windowsHide: true,
+ maxBuffer: 1024 * 1024,
+ }),
+ onProgress,
+ 'ocr_runtime_install',
+ 'Still creating local OCR Python environment',
+ );
+ onProgress?.({
+ phase: 'ocr_runtime_install',
+ message: 'Downloading and installing local OCR packages',
+ data: {
+ packages: [
+ `rapidocr==${RAPIDOCR_VERSION}`,
+ `onnxruntime==${ONNXRUNTIME_VERSION}`,
+ `pypdfium2==${PYPDFIUM2_VERSION}`,
+ ],
+ },
+ });
+ await withHeartbeat(
+ execFileAsync(
+ uv,
+ [
+ 'pip', 'install',
+ '--python', venvPython(venv),
+ '--only-binary=:all:',
+ `rapidocr==${RAPIDOCR_VERSION}`,
+ `onnxruntime==${ONNXRUNTIME_VERSION}`,
+ `pypdfium2==${PYPDFIUM2_VERSION}`,
+ ],
+ {
+ timeout: INSTALL_TIMEOUT_MS,
+ env,
+ windowsHide: true,
+ maxBuffer: 4 * 1024 * 1024,
+ },
+ ),
+ onProgress,
+ 'ocr_runtime_install',
+ 'Still installing local OCR packages',
+ );
+ onProgress?.({ phase: 'ocr_runtime_verify', message: 'Verifying local OCR runtime' });
+ } catch (err) {
+ log.warn(`install failed: ${(err as Error).message}`);
+ onProgress?.({
+ phase: 'ocr_runtime_install_failed',
+ message: 'Local OCR runtime install failed',
+ data: { error: (err as Error).message },
+ });
+ return {
+ ok: false,
+ errorCode: 'E_OCR_INSTALL_FAILED',
+ message: `Local OCR runtime install failed: ${(err as Error).message}`,
+ };
+ }
+
+ const installedVerification = await verifyRuntime(python, onProgress);
+ if (installedVerification.ok === true) {
+ onProgress?.({ phase: 'ocr_runtime_ready', message: 'Local OCR runtime installed and ready' });
+ return { ok: true, python, venv, installed: true };
+ }
+ onProgress?.({ phase: 'ocr_runtime_install_failed', message: 'Local OCR runtime verification failed' });
+ return {
+ ok: false,
+ errorCode: 'E_OCR_INSTALL_FAILED',
+ message: `Local OCR runtime installed, but verification failed: ${installedVerification.message}`,
+ };
+}
+
+async function runOcrProcess(
+ python: string,
+ payload: string,
+ onProgress: ProgressFn | undefined,
+ signal: AbortSignal | undefined,
+): Promise<string> {
+ const res = await withHeartbeat(
+ execFileAsync(python, ['-c', PYTHON_OCR_SCRIPT, payload], {
+ timeout: OCR_TIMEOUT_MS,
+ env: runtimeEnv(),
+ windowsHide: true,
+ maxBuffer: 8 * 1024 * 1024,
+ ...(signal ? { signal } : {}),
+ }),
+ onProgress,
+ 'ocr_run',
+ 'Still running local OCR',
+ );
+ return res.stdout;
+}
+
+function extKind(absPath: string): 'pdf' | 'image' | 'unsupported' {
+ const ext = path.extname(absPath).toLowerCase();
+ if (ext === '.pdf') return 'pdf';
+ if (['.png', '.jpg', '.jpeg', '.webp', '.gif'].includes(ext)) return 'image';
+ return 'unsupported';
+}
+
+function cleanPages(pages: string | undefined): string {
+ return String(pages || '').replace(/\s+/g, '').trim();
+}
+
+function fileFingerprint(absPath: string): { size: number; mtime: number } {
+ const stat = fs.statSync(absPath);
+ if (!stat.isFile()) throw new Error(`not a file: ${absPath}`);
+ return { size: stat.size, mtime: Math.floor(stat.mtimeMs) };
+}
+
+function hash(s: string): string {
+ return crypto.createHash('sha1').update(s).digest('hex');
+}
+
+function cachePaths(userId: string, absPath: string, pages: string): { dir: string; file: string } {
+ const fp = fileFingerprint(absPath);
+ const sourceHash = hash(absPath).slice(0, 16);
+ const taskHash = hash(JSON.stringify({
+ cacheVersion: CACHE_VERSION,
+ absPath,
+ size: fp.size,
+ mtime: fp.mtime,
+ pages,
+ runtime: OCR_RUNTIME_KEY,
+ })).slice(0, 16);
+ const dir = path.join(userFileCacheDir(userId), sourceHash);
+ return { dir, file: path.join(dir, `ocr.${taskHash}.md`) };
+}
+
+function formatProcessLog(processLog: string[]): string {
+ if (!processLog.length) return '';
+ return [
+ '<ocr-process>',
+ ...processLog.map((line) => `- ${line}`),
+ '</ocr-process>',
+ '',
+ ].join('\n');
+}
+
+function renderMarkdown(args: {
+ absPath: string;
+ kind: 'pdf' | 'image';
+ pages: Array<{ page: number; text: string; items: Array<{ text: string; score?: number }> }>;
+ cached: boolean;
+ processLog: string[];
+}): string {
+ const pageNums = args.pages.map((p) => p.page);
+ const pageAttr = pageNums.length ? pageNums.join(',') : '1';
+ const lines: string[] = [
+ formatProcessLog(args.processLog),
+ `<ocr-file path="${escapeXml(args.absPath)}" kind="${args.kind}" pages="${escapeXml(pageAttr)}" engine="local:rapidocr-onnxruntime" cached="${args.cached ? 'true' : 'false'}">`,
+ '',
+ ];
+ for (const p of args.pages) {
+ lines.push(`## Page ${p.page}`, '');
+ const text = p.text.trim();
+ lines.push(text || '[No text detected]');
+ lines.push('');
+ }
+ lines.push('</ocr-file>');
+ return lines.join('\n').trimEnd();
+}
+
+function escapeXml(s: string): string {
+ return s
+ .replace(/&/g, '&amp;')
+ .replace(/"/g, '&quot;')
+ .replace(/</g, '&lt;')
+ .replace(/>/g, '&gt;');
+}
+
+export async function ocrFile(input: OcrFileInput): Promise<OcrFileResult> {
+ const processLog: string[] = [];
+ const emitProgress: ProgressFn = (event) => {
+ const suffix = event.data?.seconds ? ` (${event.data.seconds}s)` : '';
+ processLog.push(`${event.message}${suffix}`);
+ input.onProgress?.(event);
+ };
+ const kind = extKind(input.absPath);
+ if (kind === 'unsupported') {
+ return {
+ ok: false,
+ errorCode: 'E_OCR_UNSUPPORTED_FILE',
+ message: 'ocr_file currently supports PDF and image files only. Use read_file for normal text/Office files.',
+ processLog,
+ };
+ }
+
+ let pages = cleanPages(input.pages);
+ const { file: cacheFile, dir: cacheDir } = cachePaths(input.userId, input.absPath, pages);
+ try {
+ if (fs.existsSync(cacheFile)) {
+ const content = fs.readFileSync(cacheFile, 'utf8')
+ .replace(/<ocr-process>[\s\S]*?<\/ocr-process>\s*/m, '')
+ .replace('cached="false"', 'cached="true"');
+ processLog.push('Using cached OCR result');
+ return {
+ ok: true,
+ content: `${formatProcessLog(processLog)}${content}`,
+ pages: [],
+ cached: true,
+ engine: 'local:rapidocr-onnxruntime',
+ processLog,
+ };
+ }
+ } catch { /* best effort */ }
+
+ emitProgress({ phase: 'ocr_prepare', message: 'Preparing local OCR runtime' });
+ const runtime = await ensureRuntime(emitProgress);
+ if (runtime.ok === false) {
+ return { ok: false, errorCode: runtime.errorCode, message: runtime.message, processLog };
+ }
+
+ emitProgress({ phase: 'ocr_run', message: kind === 'pdf' ? 'Rendering PDF pages and running OCR' : 'Running OCR' });
+ const payload = JSON.stringify({
+ path: input.absPath,
+ kind,
+ pages,
+ scale: 2,
+ });
+
+ let stdout = '';
+ try {
+ stdout = await runOcrProcess(runtime.python, payload, emitProgress, input.signal);
+ } catch (err) {
+ log.warn(`ocr failed path=${input.absPath}: ${(err as Error).message}`);
+ processLog.push('Local OCR process failed');
+ return {
+ ok: false,
+ errorCode: 'E_OCR_FAILED',
+ message: `Local OCR failed: ${(err as Error).message}`,
+ processLog,
+ };
+ }
+
+ try {
+ const parsed = JSON.parse(stdout);
+ if (!parsed?.ok) {
+ processLog.push('Local OCR returned an error');
+ return {
+ ok: false,
+ errorCode: 'E_OCR_FAILED',
+ message: String(parsed?.error || 'Local OCR failed'),
+ processLog,
+ };
+ }
+ const pageResults = Array.isArray(parsed.pages) ? parsed.pages : [];
+ processLog.push(`OCR completed for ${pageResults.length || 0} page${pageResults.length === 1 ? '' : 's'}`);
+ const content = renderMarkdown({ absPath: input.absPath, kind, pages: pageResults, cached: false, processLog });
+ fs.mkdirSync(cacheDir, { recursive: true });
+ fs.writeFileSync(cacheFile, content, 'utf8');
+ return {
+ ok: true,
+ content,
+ pages: pageResults.map((p: { page: number }) => p.page).filter((n: unknown): n is number => typeof n === 'number'),
+ cached: false,
+ engine: 'local:rapidocr-onnxruntime',
+ processLog,
+ };
+ } catch (err) {
+ processLog.push('Local OCR returned invalid output');
+ return {
+ ok: false,
+ errorCode: 'E_OCR_FAILED',
+ message: `Local OCR returned invalid output: ${(err as Error).message}`,
+ processLog,
+ };
+ }
+}
+
+// Keep this script dependency-light and resilient across RapidOCR package API
+// changes. The pinned `rapidocr` package needs a string model_root_dir here:
+// passing its default pathlib.Path trips OmegaConf on some Python versions.
+const PYTHON_OCR_SCRIPT = String.raw`
+import json, os, pathlib, sys, tempfile, traceback
+
+def load_engine():
+ try:
+ import rapidocr as rapidocr_pkg
+ from rapidocr import RapidOCR
+ except ModuleNotFoundError as exc:
+ if exc.name != "rapidocr":
+ raise
+ try:
+ from rapidocr_onnxruntime import RapidOCR
+ return RapidOCR()
+ except ModuleNotFoundError:
+ raise exc
+ model_dir = pathlib.Path(rapidocr_pkg.__file__).resolve().parent / "models"
+ return RapidOCR(params={"Global.model_root_dir": str(model_dir)})
+
+def parse_pages(spec, total):
+ if not spec:
+ return list(range(1, total + 1))
+ out = []
+ for part in str(spec).split(","):
+ if not part:
+ continue
+ if "-" in part:
+ a, b = part.split("-", 1)
+ start, end = int(a), int(b)
+ if start > end:
+ start, end = end, start
+ out.extend(range(start, end + 1))
+ else:
+ out.append(int(part))
+ seen = []
+ for n in out:
+ if n < 1 or n > total:
+ raise ValueError(f"page {n} out of range 1-{total}")
+ if n not in seen:
+ seen.append(n)
+ return seen
+
+def item_text(item):
+ if isinstance(item, dict):
+ txt = item.get("text") or item.get("txt") or item.get("label")
+ score = item.get("score") or item.get("confidence")
+ return txt, score
+ if isinstance(item, (list, tuple)):
+ if len(item) >= 2 and isinstance(item[1], str):
+ return item[1], item[2] if len(item) >= 3 and isinstance(item[2], (int, float)) else None
+ if len(item) >= 1 and isinstance(item[0], str):
+ return item[0], item[1] if len(item) >= 2 and isinstance(item[1], (int, float)) else None
+ if isinstance(item, str):
+ return item, None
+ return None, None
+
+def collect_result(result):
+ if isinstance(result, tuple) and result:
+ # rapidocr_onnxruntime returns (results, elapsed)
+ result = result[0]
+ if hasattr(result, "txts"):
+ txts = list(getattr(result, "txts") or [])
+ scores = list(getattr(result, "scores", []) or [])
+ return [{"text": str(t), **({"score": float(scores[i])} if i < len(scores) and isinstance(scores[i], (int, float)) else {})} for i, t in enumerate(txts) if str(t).strip()]
+ if hasattr(result, "to_json"):
+ try:
+ return collect_result(json.loads(result.to_json()))
+ except Exception:
+ pass
+ if isinstance(result, dict):
+ if "txts" in result:
+ txts = result.get("txts") or []
+ scores = result.get("scores") or []
+ return [{"text": str(t), **({"score": float(scores[i])} if i < len(scores) and isinstance(scores[i], (int, float)) else {})} for i, t in enumerate(txts) if str(t).strip()]
+ for key in ("data", "result", "results"):
+ if key in result:
+ return collect_result(result[key])
+ if isinstance(result, list):
+ out = []
+ for item in result:
+ if isinstance(item, (list, tuple, dict, str)):
+ txt, score = item_text(item)
+ if txt and str(txt).strip():
+ obj = {"text": str(txt)}
+ if isinstance(score, (int, float)):
+ obj["score"] = float(score)
+ out.append(obj)
+ elif not isinstance(item, str):
+ out.extend(collect_result(item))
+ return out
+ return []
+
+def run_ocr(engine, image_path):
+ result = engine(image_path)
+ items = collect_result(result)
+ return {
+ "text": "\n".join([i["text"] for i in items if i.get("text")]).strip(),
+ "items": items,
+ }
+
+def main():
+ args = json.loads(sys.argv[1])
+ src = args["path"]
+ kind = args["kind"]
+ engine = load_engine()
+ pages = []
+ if kind == "image":
+ r = run_ocr(engine, src)
+ pages.append({"page": 1, **r})
+ elif kind == "pdf":
+ import pypdfium2 as pdfium
+ pdf = pdfium.PdfDocument(src)
+ page_nums = parse_pages(args.get("pages") or "", len(pdf))
+ scale = float(args.get("scale") or 2)
+ with tempfile.TemporaryDirectory(prefix="orkas-ocr-") as td:
+ for pno in page_nums:
+ page = pdf.get_page(pno - 1)
+ try:
+ try:
+ pil = page.render(scale=scale).to_pil()
+ except AttributeError:
+ pil = page.render_topil(scale=scale)
+ out = os.path.join(td, f"page-{pno}.png")
+ pil.save(out, "PNG")
+ r = run_ocr(engine, out)
+ pages.append({"page": pno, **r})
+ finally:
+ try:
+ page.close()
+ except Exception:
+ pass
+ else:
+ raise ValueError(f"unsupported kind: {kind}")
+ print(json.dumps({"ok": True, "pages": pages}, ensure_ascii=False))
+
+try:
+ main()
+except Exception as exc:
+ print(json.dumps({"ok": False, "error": str(exc), "trace": traceback.format_exc()}, ensure_ascii=False))
+`;
diff --git a/src/main/model/core-agent/file-tools.ts b/src/main/model/core-agent/file-tools.ts
index 8d160c28..265c685b 100644
--- a/src/main/model/core-agent/file-tools.ts
+++ b/src/main/model/core-agent/file-tools.ts
@@ -45,6 +45,7 @@ import {
NoTextError,
UnsupportedFileKindError,
} from '../../features/file_indexer';
+import { ocrFile } from '../../features/ocr_runtime';
import { chatAttachmentDir, userMarketplaceSkillsDir, userSkillsDir } from '../../paths';
import { getWorkspacePath } from '../../features/user_workspace';
import { isPathAllowed } from '../../util/path-sandbox';
@@ -438,6 +439,86 @@ function createStatFileTool(opts: FileToolsOpts): AgentTool {
};
}
+// ── ocr_file ─────────────────────────────────────────────────────────────
+
+function createOcrFileTool(opts: FileToolsOpts): AgentTool {
+ return {
+ name: 'ocr_file',
+ executionMode: 'sequential',
+ description:
+ 'Run local OCR on visual text in a file and return Markdown. Use this when read_file/stat_file cannot recover text from visual content: scanned PDFs, image-only PDF pages, screenshots, photos of documents, or text embedded in images. For normal text PDFs/Office files, use stat_file/read_file first. For mixed PDFs, combine read_file text with ocr_file visual text.\n'
+ + '\n'
+ + 'Parameters:\n'
+ + ' path — required. Absolute path inside the current workspace or this conversation\'s attachment dir.\n'
+ + ' pages — optional for PDFs. Page list/ranges like "1-3,5". Omit to OCR all pages.\n'
+ + '\n'
+ + 'Supported inputs in this version: PDF pages and image files (.png/.jpg/.jpeg/.webp/.gif). Office documents should be read with read_file; embedded-image OCR for Office is not supported yet.\n'
+ + 'The OCR engine is local RapidOCR + ONNXRuntime and does not consume cloud model credits. If the local OCR runtime is not installed, this tool installs it into Orkas\'s managed runtime directory. If this tool returns an E_OCR_* error, do not try to install or repair OCR dependencies with bash/pip/uv; report the tool error and process log.',
+ inputSchema: {
+ type: 'object',
+ properties: {
+ path: { type: 'string', description: 'Absolute path. Must be inside workspace or current attachment dir.' },
+ pages: { type: 'string', description: 'Optional PDF pages, e.g. "1-3,5". Omit to OCR all pages.' },
+ },
+ required: ['path'],
+ },
+ async execute(input, ctx) {
+ const raw = String(input.path ?? '');
+ if (!raw) return { content: errText('E_BAD_INPUT', '`path` is required'), isError: true };
+ const abs = resolveAbs(ctx, raw);
+
+ const scopeErr = guardPath(opts, abs);
+ if (scopeErr) {
+ log.warn(`ocr_file scope reject user=${opts.userId} path=${abs}`);
+ return { content: scopeErr, isError: true };
+ }
+ const disabledSkillErr = guardDisabledSkillAccess(opts, abs);
+ if (disabledSkillErr) {
+ log.warn(`ocr_file disabled skill reject user=${opts.userId} path=${abs}`);
+ return { content: disabledSkillErr, isError: true };
+ }
+ try { fs.statSync(abs); }
+ catch (err) {
+ log.warn(`ocr_file not-found user=${opts.userId} path=${abs}: ${(err as Error).message}`);
+ return { content: errText('E_NOT_FOUND', `${abs}: ${(err as Error).message}`), isError: true };
+ }
+
+ const kind = kindOf(abs);
+ if (kind !== 'pdf' && kind !== 'image') {
+ return {
+ content: errText(
+ 'E_OCR_UNSUPPORTED_FILE',
+ `ocr_file currently supports PDF and image files only; got kind=${kind}. Use read_file/stat_file for normal text or Office files.`,
+ ),
+ isError: true,
+ };
+ }
+
+ const pages = typeof input.pages === 'string' ? input.pages : undefined;
+ const result = await ocrFile({
+ userId: opts.userId,
+ absPath: abs,
+ ...(pages ? { pages } : {}),
+ ...(ctx.signal ? { signal: ctx.signal } : {}),
+ onProgress: (event) => ctx.emitProgress?.({
+ phase: event.phase,
+ message: event.message,
+ ...(event.data ? { data: event.data } : {}),
+ }),
+ });
+ if (result.ok === false) {
+ const processBlock = result.processLog?.length
+ ? `\n\n<ocr-process>\n${result.processLog.map((line) => `- ${line}`).join('\n')}\n</ocr-process>`
+ : '';
+ const repairHint = '\n\nDo not install or repair OCR dependencies with bash/pip/uv; ocr_file owns its local runtime.';
+ return { content: errText(result.errorCode, `${result.message}${processBlock}${repairHint}`), isError: true };
+ }
+ log.info(`ocr_file user=${opts.userId} kind=${kind} cached=${result.cached} path=${abs}`);
+ return { content: result.content };
+ },
+ };
+}
+
// ── search_files ─────────────────────────────────────────────────────────
interface SearchHit {
@@ -934,6 +1015,7 @@ export function createFileTools(opts: FileToolsOpts): AgentTool[] {
return [
createReadFileTool(opts),
createStatFileTool(opts),
+ createOcrFileTool(opts),
createSearchFilesTool(opts),
createGrepFilesTool(opts),
createListFilesTool(opts),
diff --git a/src/main/model/core-agent/tool-catalog.ts b/src/main/model/core-agent/tool-catalog.ts
index e7de8e63..17990551 100644
--- a/src/main/model/core-agent/tool-catalog.ts
+++ b/src/main/model/core-agent/tool-catalog.ts
@@ -69,6 +69,7 @@ export const TOOL_CATALOG: ToolCatalogEntry[] = [
{ name: 'delete_file', group: 'fs', permission: 'localExec', summary: 'Delete a single file from the workspace / current attachment dir / extraRoots. The first call shows an inline confirmation card and returns a token; multiple deletes from the same turn are grouped in one card when possible. Use instead of `bash rm` for removals.' },
{ name: 'list_files', group: 'fs', summary: 'List the workspace directory tree.' },
{ name: 'stat_file', group: 'fs', summary: 'Trigger PDF/modern Office extraction and return total_chars; call before read_file.' },
+ { name: 'ocr_file', group: 'fs', summary: 'Run local OCR on PDF pages or image files when visual text is not available through read_file/stat_file.' },
{ name: 'search_files', group: 'fs', summary: 'Find files by name / glob across the workspace + attachment scope.' },
{ name: 'grep_files', group: 'fs', summary: 'Grep text across the workspace + attachment scope (PDF/modern Office auto-extracted, then searched).' },
{ name: 'create_artifact', group: 'fs', permission: 'localExec', summary: 'Build an interactive multi-file app (HTML/CSS/JS) rendered live & clickable inside the chat bubble; for interactive dashboards / calculators / visualizations / mini-tools. Static/read-only dashboards should use :::dashboard; not documents (html_to_pdf) or images (generate_image).' },
diff --git a/test/main/model/core-agent/file-tools.test.ts b/test/main/model/core-agent/file-tools.test.ts
index 3d783a79..2d5cede7 100644
--- a/test/main/model/core-agent/file-tools.test.ts
+++ b/test/main/model/core-agent/file-tools.test.ts
@@ -28,6 +28,8 @@ beforeEach(async () => {
});
afterEach(() => {
+ vi.doUnmock('../../../../src/main/features/ocr_runtime');
+ vi.restoreAllMocks();
process.env.ORKAS_WORKSPACE_ROOT = prevWs;
if (prevHome === undefined) delete process.env.HOME;
else process.env.HOME = prevHome;
@@ -211,6 +213,92 @@ describe('file-tools › read_file (image)', () => {
});
});
+describe('file-tools › ocr_file', () => {
+ it('runs local OCR for images and returns OCR markdown', async () => {
+ const mockOcr = vi.fn(async () => ({
+ ok: true,
+ content: '<ocr-file path="/x" kind="image" pages="1" engine="local:rapidocr-onnxruntime" cached="false">\nhello\n</ocr-file>',
+ pages: [1],
+ cached: false,
+ engine: 'local:rapidocr-onnxruntime',
+ }));
+ vi.doMock('../../../../src/main/features/ocr_runtime', () => ({ ocrFile: mockOcr }));
+ const { tools, wsDir } = await buildTools();
+ const p = path.join(wsDir, 'scan.png');
+ const { Jimp } = await import('jimp' as any);
+ const img: any = new Jimp({ width: 20, height: 20, color: 0xFFFFFFFF });
+ fs.writeFileSync(p, await img.getBuffer('image/png'));
+
+ const r = await run(getTool(tools, 'ocr_file'), { path: p });
+
+ expect(r.isError).toBeFalsy();
+ expect(r.content).toContain('hello');
+ expect(mockOcr).toHaveBeenCalledWith(expect.objectContaining({
+ userId: UID,
+ absPath: p,
+ }));
+ });
+
+ it('passes PDF page ranges to the local OCR runtime', async () => {
+ const mockOcr = vi.fn(async () => ({
+ ok: true,
+ content: '<ocr-file path="/x" kind="pdf" pages="1,3" engine="local:rapidocr-onnxruntime" cached="false">\npage text\n</ocr-file>',
+ pages: [1, 3],
+ cached: false,
+ engine: 'local:rapidocr-onnxruntime',
+ }));
+ vi.doMock('../../../../src/main/features/ocr_runtime', () => ({ ocrFile: mockOcr }));
+ const { tools, wsDir } = await buildTools();
+ const p = path.join(wsDir, 'scan.pdf');
+ fs.writeFileSync(p, makeMinimalPdf(['']));
+
+ const r = await run(getTool(tools, 'ocr_file'), { path: p, pages: '1,3' });
+
+ expect(r.isError).toBeFalsy();
+ expect(mockOcr).toHaveBeenCalledWith(expect.objectContaining({ absPath: p, pages: '1,3' }));
+ });
+
+ it('surfaces local OCR runtime errors with process info', async () => {
+ vi.doMock('../../../../src/main/features/ocr_runtime', () => ({
+ ocrFile: vi.fn(async () => ({
+ ok: false,
+ errorCode: 'E_OCR_INSTALL_FAILED',
+ message: 'Local OCR runtime install failed.',
+ processLog: [
+ 'Preparing local OCR runtime',
+ 'Checking local OCR runtime',
+ 'Downloading and installing local OCR packages',
+ ],
+ })),
+ }));
+ const { tools, wsDir } = await buildTools();
+ const p = path.join(wsDir, 'scan.pdf');
+ fs.writeFileSync(p, makeMinimalPdf(['']));
+
+ const r = await run(getTool(tools, 'ocr_file'), { path: p });
+
+ expect(r.isError).toBe(true);
+ expect(r.content).toContain('E_OCR_INSTALL_FAILED');
+ expect(r.content).toContain('Local OCR runtime install failed');
+ expect(r.content).toContain('<ocr-process>');
+ expect(r.content).toContain('Downloading and installing local OCR packages');
+ });
+
+ it('rejects unsupported file kinds before invoking OCR runtime', async () => {
+ const mockOcr = vi.fn();
+ vi.doMock('../../../../src/main/features/ocr_runtime', () => ({ ocrFile: mockOcr }));
+ const { tools, wsDir } = await buildTools();
+ const p = path.join(wsDir, 'notes.docx');
+ fs.writeFileSync(p, makeMinimalDocx({ paragraphs: ['not visual'] }));
+
+ const r = await run(getTool(tools, 'ocr_file'), { path: p });
+
+ expect(r.isError).toBe(true);
+ expect(r.content).toContain('E_OCR_UNSUPPORTED_FILE');
+ expect(mockOcr).not.toHaveBeenCalled();
+ });
+});
+
describe('file-tools › read_file scope guards', () => {
it('rejects paths outside the scope with E_PATH_OUT_OF_SCOPE', async () => {
const { tools } = await buildTools();