Orkas is an open-source (MIT), local-first AI team for your desktop: a Commander turns your goal into to-dos and coordinates 16 built-in specialist agents plus Claude Code, Codex, OpenCode, OpenClaw and Hermes, with projects, scheduled automations and connectors. Free app for macOS & Windows; built-in models or your own keys.
JavaScript agent-collaborationagent-harnessagent-orchestrationagent-teamagent-workflowai-assistantai-automationai-employeeai-productivityai-teamai-workflowautonomous-agentsclaude-code-guiclaude-coworkdesktop-agentlocal-firstmulti-agentmulti-agent-systemself-evolving-agentstask-automation
Add VideoStudio runtime tool
wytd005cbc14ec6
diff --git a/src/main/features/hyperframes_runtime.ts b/src/main/features/hyperframes_runtime.tsindex 7cfb528f..1573a25a 100644--- a/src/main/features/hyperframes_runtime.ts+++ b/src/main/features/hyperframes_runtime.ts@@ -9,6 +9,7 @@*/import { spawn } from 'node:child_process';+import * as fs from 'node:fs';import {bundledNodeExecutable,@@ -46,6 +47,9 @@ export function buildHyperframesEnv(): NodeJS.ProcessEnv {if (ff.ffmpeg) env.HYPERFRAMES_FFMPEG_PATH = ff.ffmpeg;if (ff.ffprobe) env.HYPERFRAMES_FFPROBE_PATH = ff.ffprobe;+ for (const dir of [paths.NODE_NPM_CACHE_DIR, paths.NODE_NPM_PREFIX_DIR, paths.NODE_NPM_GLOBAL_BIN_DIR]) {+ try { fs.mkdirSync(dir, { recursive: true }); } catch { /* npm will surface a clearer error later */ }+ }env.NPM_CONFIG_CACHE = paths.NODE_NPM_CACHE_DIR;env.NPM_CONFIG_PREFIX = paths.NODE_NPM_PREFIX_DIR;env.NPM_CONFIG_FUND = 'false';diff --git a/src/main/features/video_studio.ts b/src/main/features/video_studio.tsnew file mode 100644index 00000000..bbaa0101--- /dev/null+++ b/src/main/features/video_studio.ts@@ -0,0 +1,832 @@+import * as fs from 'node:fs/promises';+import * as fss from 'node:fs';+import * as os from 'node:os';+import * as path from 'node:path';+import { spawn } from 'node:child_process';+import type { BrowserWindow as ElectronBrowserWindow } from 'electron';++import { bundledFfmpegPaths } from '../util/bundled-runtime';+import { createLogger } from '../logger';++const log = createLogger('video-studio');++export type VideoStudioOp =+ | 'composition.lint'+ | 'composition.inspect'+ | 'composition.render'+ | 'composition.draft'+ | 'composition.snapshot'+ | 'speech.transcribe';++export type RenderQuality = 'draft' | 'standard' | 'high';+export type RenderFormat = 'mp4' | 'webm';++export interface CompositionOptions {+ compositionDirAbs: string;+ outputAbsPath?: string;+ reportAbsPath?: string;+ findingsAbsPath?: string;+ snapshotAbsPath?: string;+ quality?: RenderQuality;+ fps?: number;+ format?: RenderFormat;+ variables?: Record<string, unknown>;+ signal?: AbortSignal;+ onProgress?: (event: { phase: string; message: string; data?: Record<string, unknown> }) => void;+}++export interface SpeechTranscribeOptions {+ inputAbsPath: string;+ transcriptAbsPath?: string;+ model?: string;+ language?: string;+ timestamps?: 'segment' | 'word';+ allowModelDownload?: boolean;+ signal?: AbortSignal;+ onProgress?: (event: { phase: string; message: string; data?: Record<string, unknown> }) => void;+}++export type VideoStudioResult =+ | { ok: true; op: VideoStudioOp; [key: string]: unknown }+ | { ok: false; op?: VideoStudioOp; errorCode: string; message: string; [key: string]: unknown };++type Issue = {+ code: string;+ severity: 'error' | 'warning' | 'info';+ selector?: string;+ message: string;+ fixHint?: string;+ source?: string;+};++type CompositionMeta = {+ htmlPath: string;+ html: string;+ rootAttrs: Record<string, string>;+ id: string;+ width: number;+ height: number;+ durationSec: number;+ audioTracks: Array<{ absPath: string; startSec: number }>;+};++const DEFAULT_WIDTH = 1920;+const DEFAULT_HEIGHT = 1080;+const DEFAULT_DURATION_SEC = 5;+const MAX_RENDER_DURATION_SEC = 20 * 60;+const MAX_FPS = 60;+const RENDER_TIMEOUT_MS = 20 * 60 * 1000;+function round2(n: number): number {+ return Math.round((Number.isFinite(n) ? n : 0) * 100) / 100;+}++function isFinitePositive(n: unknown): n is number {+ return typeof n === 'number' && Number.isFinite(n) && n > 0;+}++function parseAttrs(tag: string): Record<string, string> {+ const attrs: Record<string, string> = {};+ const re = /([:\w-]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+))/g;+ let m: RegExpExecArray | null;+ while ((m = re.exec(tag)) !== null) {+ attrs[m[1].toLowerCase()] = m[2] ?? m[3] ?? m[4] ?? '';+ }+ return attrs;+}++function htmlAttrNumber(attrs: Record<string, string>, key: string): number {+ const v = Number(attrs[key]);+ return Number.isFinite(v) && v > 0 ? v : 0;+}++function normalizeRef(ref: string): string {+ return String(ref || '').trim().replace(/&/g, '&');+}++function isRemoteRef(ref: string): boolean {+ return /^(?:https?:)?\/\//i.test(ref);+}++function isIgnorableRef(ref: string): boolean {+ return !ref+ || ref.startsWith('#')+ || /^data:/i.test(ref)+ || /^blob:/i.test(ref)+ || /^javascript:/i.test(ref)+ || /^mailto:/i.test(ref);+}++function safeResolveLocalRef(compositionDirAbs: string, ref: string): string | null {+ const noHash = normalizeRef(ref).split('#')[0].split('?')[0];+ if (isIgnorableRef(noHash) || isRemoteRef(noHash) || path.isAbsolute(noHash)) return null;+ let decoded = noHash;+ try { decoded = decodeURIComponent(noHash); } catch { /* keep raw */ }+ const abs = path.resolve(compositionDirAbs, decoded);+ const rel = path.relative(compositionDirAbs, abs);+ if (!rel || rel.startsWith('..') || path.isAbsolute(rel)) return null;+ return abs;+}++function extractResourceRefs(html: string): Array<{ attr: string; ref: string }> {+ const refs: Array<{ attr: string; ref: string }> = [];+ const re = /\b(src|href|poster)\s*=\s*(?:"([^"]*)"|'([^']*)')/gi;+ let m: RegExpExecArray | null;+ while ((m = re.exec(html)) !== null) {+ refs.push({ attr: m[1].toLowerCase(), ref: normalizeRef(m[2] ?? m[3] ?? '') });+ }+ return refs;+}++function findingsJson(issues: Issue[], extra: Record<string, unknown> = {}): string {+ const errorCount = issues.filter((i) => i.severity === 'error').length;+ const warningCount = issues.filter((i) => i.severity === 'warning').length;+ return JSON.stringify({+ ok: errorCount === 0,+ errorCount,+ warningCount,+ issueCount: issues.length,+ totalIssueCount: issues.length,+ issues,+ ...extra,+ }, null, 2);+}++function qualityFps(quality: RenderQuality | undefined, fps: number | undefined): number {+ if (isFinitePositive(fps)) return Math.max(1, Math.min(MAX_FPS, Math.floor(fps)));+ if (quality === 'draft') return 15;+ return 30;+}++function crfForQuality(quality: RenderQuality | undefined): number {+ if (quality === 'high') return 16;+ if (quality === 'standard') return 20;+ return 26;+}++async function loadCompositionMeta(compositionDirAbs: string): Promise<{ meta: CompositionMeta | null; issues: Issue[] }> {+ const issues: Issue[] = [];+ const htmlPath = path.join(compositionDirAbs, 'index.html');+ const st = await fs.stat(htmlPath).catch(() => null);+ if (!st || !st.isFile()) {+ return {+ meta: null,+ issues: [{+ code: 'NO_COMPOSITION',+ severity: 'error',+ selector: 'index.html',+ message: `No index.html found in composition dir: ${compositionDirAbs}`,+ }],+ };+ }++ const html = await fs.readFile(htmlPath, 'utf8');+ const rootTag = html.match(/<[^>]+\bdata-composition-id\s*=\s*["'][^"']+["'][^>]*>/i)?.[0] ?? '';+ const rootAttrs = rootTag ? parseAttrs(rootTag) : {};+ const width = htmlAttrNumber(rootAttrs, 'data-width') || DEFAULT_WIDTH;+ const height = htmlAttrNumber(rootAttrs, 'data-height') || DEFAULT_HEIGHT;+ const durationSec = htmlAttrNumber(rootAttrs, 'data-duration') || DEFAULT_DURATION_SEC;+ const id = rootAttrs['data-composition-id'] || 'main';++ if (!rootTag) {+ issues.push({+ code: 'ROOT_COMPOSITION_MISSING',+ severity: 'error',+ selector: '[data-composition-id]',+ message: 'index.html must declare a root element with data-composition-id, data-width, data-height, and data-duration.',+ });+ }+ for (const key of ['data-width', 'data-height', 'data-duration']) {+ if (!htmlAttrNumber(rootAttrs, key)) {+ issues.push({+ code: 'ROOT_TIMING_ATTR_MISSING',+ severity: 'error',+ selector: '[data-composition-id]',+ message: `root composition is missing a positive numeric ${key}.`,+ });+ }+ }+ if (durationSec > MAX_RENDER_DURATION_SEC) {+ issues.push({+ code: 'DURATION_TOO_LONG',+ severity: 'error',+ selector: '[data-composition-id]',+ message: `composition duration ${durationSec}s exceeds the ${MAX_RENDER_DURATION_SEC}s render limit.`,+ });+ }++ const refs = extractResourceRefs(html);+ const audioTracks: Array<{ absPath: string; startSec: number }> = [];+ for (const item of refs) {+ if (isIgnorableRef(item.ref)) continue;+ if (isRemoteRef(item.ref)) {+ issues.push({+ code: 'REMOTE_RESOURCE_BLOCKED',+ severity: 'error',+ selector: `[${item.attr}="${item.ref}"]`,+ message: `Remote runtime resource is not allowed during video render: ${item.ref}`,+ fixHint: 'Copy runtime assets into the composition directory and reference them relatively.',+ });+ continue;+ }+ if (path.isAbsolute(item.ref)) {+ issues.push({+ code: 'ABSOLUTE_RESOURCE_BLOCKED',+ severity: 'error',+ selector: `[${item.attr}="${item.ref}"]`,+ message: `Absolute runtime resource is not allowed during video render: ${item.ref}`,+ });+ continue;+ }+ const abs = safeResolveLocalRef(compositionDirAbs, item.ref);+ if (!abs) {+ issues.push({+ code: 'RESOURCE_OUT_OF_SCOPE',+ severity: 'error',+ selector: `[${item.attr}="${item.ref}"]`,+ message: `Resource reference escapes the composition directory: ${item.ref}`,+ });+ continue;+ }+ const exists = await fs.stat(abs).catch(() => null);+ if (!exists || !exists.isFile()) {+ issues.push({+ code: 'LOCAL_RESOURCE_MISSING',+ severity: 'error',+ selector: `[${item.attr}="${item.ref}"]`,+ message: `Local resource does not exist: ${item.ref}`,+ });+ }+ }++ const audioRe = /<audio\b[^>]*>/gi;+ let audioMatch: RegExpExecArray | null;+ while ((audioMatch = audioRe.exec(html)) !== null) {+ const attrs = parseAttrs(audioMatch[0]);+ const src = attrs.src;+ if (!src || isIgnorableRef(src) || isRemoteRef(src) || path.isAbsolute(src)) continue;+ const abs = safeResolveLocalRef(compositionDirAbs, src);+ if (abs) audioTracks.push({ absPath: abs, startSec: Number(attrs['data-start']) || 0 });+ }++ return {+ meta: { htmlPath, html, rootAttrs, id, width, height, durationSec, audioTracks },+ issues,+ };+}++export async function lintComposition(p: CompositionOptions): Promise<VideoStudioResult> {+ const { meta, issues } = await loadCompositionMeta(p.compositionDirAbs);+ const findings = findingsJson(issues, {+ engine: 'orkas-native',+ profile: 'orkas-html-composition',+ canvas: meta ? { width: meta.width, height: meta.height, durationSec: meta.durationSec } : null,+ });+ return { ok: true, op: 'composition.lint', findings };+}++function fileUrl(absPath: string): string {+ let resolved = path.resolve(absPath).replace(/\\/g, '/');+ if (!resolved.startsWith('/')) resolved = `/${resolved}`;+ return encodeURI(`file://${resolved}`);+}++async function withCompositionWindow<T>(+ meta: CompositionMeta,+ p: CompositionOptions,+ fn: (win: ElectronBrowserWindow) => Promise<T>,+): Promise<T> {+ const electron = await import('electron');+ const { BrowserWindow, session } = electron;+ if (!BrowserWindow) throw new Error('Electron BrowserWindow unavailable');+ const partition = `video-studio-${Date.now()}-${Math.random().toString(36).slice(2)}`;+ const ses = session.fromPartition(partition);+ ses.webRequest.onBeforeRequest((details, callback) => {+ const url = String(details.url || '');+ if (/^https?:\/\//i.test(url)) {+ callback({ cancel: true });+ return;+ }+ callback({});+ });++ const win = new BrowserWindow({+ show: false,+ width: meta.width,+ height: meta.height,+ useContentSize: true,+ backgroundColor: '#000000',+ webPreferences: {+ sandbox: true,+ contextIsolation: true,+ nodeIntegration: false,+ webSecurity: true,+ session: ses,+ },+ });+ try {+ const loaded = new Promise<void>((resolve, reject) => {+ win.webContents.once('did-finish-load', () => resolve());+ win.webContents.once('did-fail-load', (_e, _code, desc) => reject(new Error(`did-fail-load: ${desc}`)));+ });+ await win.loadURL(fileUrl(meta.htmlPath));+ await loaded;+ await win.webContents.executeJavaScript(buildTimelineAdapterScript(meta, p.variables), true);+ await waitForReady(win);+ return await fn(win);+ } finally {+ try { win.destroy(); } catch { /* best effort */ }+ }+}++function buildTimelineAdapterScript(meta: CompositionMeta, variables?: Record<string, unknown>): string {+ const vars = JSON.stringify(variables || {});+ return `+(() => {+ const root = document.querySelector('[data-composition-id]') || document.body;+ const compositionId = root && root.getAttribute ? (root.getAttribute('data-composition-id') || ${JSON.stringify(meta.id)}) : ${JSON.stringify(meta.id)};+ const variables = ${vars};+ window.__ORKAS_VIDEO_VARIABLES__ = variables;+ function num(v, fallback) {+ const n = Number(v);+ return Number.isFinite(n) ? n : fallback;+ }+ function pauseMedia() {+ for (const media of Array.from(document.querySelectorAll('audio,video'))) {+ try { media.pause(); } catch {}+ }+ }+ async function seekMedia(t) {+ const mediaEls = Array.from(document.querySelectorAll('audio,video'));+ await Promise.all(mediaEls.map((media) => new Promise((resolve) => {+ const start = num(media.getAttribute('data-start'), 0);+ const local = Math.max(0, t - start);+ let done = false;+ const finish = () => { if (!done) { done = true; resolve(); } };+ try {+ media.pause();+ if (Number.isFinite(media.duration) && Math.abs(media.currentTime - local) > 0.04) {+ media.addEventListener('seeked', finish, { once: true });+ media.currentTime = Math.min(local, Math.max(0, media.duration || local));+ setTimeout(finish, 1000);+ } else {+ finish();+ }+ } catch {+ finish();+ }+ })));+ }+ function seekTimelines(t) {+ const timelines = window.__timelines || {};+ const tl = timelines[compositionId] || timelines.main || timelines.root;+ if (!tl) return false;+ try {+ if (typeof tl.pause === 'function') tl.pause();+ if (typeof tl.seek === 'function') { tl.seek(t, false); return true; }+ if (typeof tl.time === 'function') { tl.time(t, false); return true; }+ if (typeof tl.progress === 'function' && typeof tl.duration === 'function') {+ const dur = Number(tl.duration()) || ${meta.durationSec};+ tl.progress(dur > 0 ? Math.max(0, Math.min(1, t / dur)) : 0, false);+ return true;+ }+ } catch {}+ return false;+ }+ function seekWebAnimations(t) {+ try {+ for (const anim of document.getAnimations({ subtree: true })) {+ try {+ anim.pause();+ anim.currentTime = Math.max(0, t * 1000);+ } catch {}+ }+ } catch {}+ }+ function applyTimedVisibility(t) {+ for (const el of Array.from(document.querySelectorAll('[data-start], [data-duration]'))) {+ if (el === root) continue;+ const start = num(el.getAttribute('data-start'), 0);+ const dur = num(el.getAttribute('data-duration'), Number.POSITIVE_INFINITY);+ if (!Number.isFinite(dur)) continue;+ const visible = t >= start && t <= start + dur;+ if (el.classList && el.classList.contains('clip')) {+ el.style.visibility = visible ? '' : 'hidden';+ }+ }+ }+ window.__ORKAS_VIDEO__ = window.__ORKAS_VIDEO__ || {};+ window.__ORKAS_VIDEO__.duration = ${meta.durationSec};+ window.__ORKAS_VIDEO__.seek = async (t) => {+ pauseMedia();+ const usedTimeline = seekTimelines(t);+ seekWebAnimations(t);+ if (!usedTimeline) applyTimedVisibility(t);+ await seekMedia(t);+ await new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve)));+ };+ pauseMedia();+})()+`;+}++async function waitForReady(win: ElectronBrowserWindow): Promise<void> {+ await win.webContents.executeJavaScript(`+(async () => {+ try { if (document.fonts && document.fonts.ready) await document.fonts.ready; } catch {}+ const imgs = Array.from(document.images || []);+ await Promise.all(imgs.map((img) => {+ if (img.complete) return Promise.resolve();+ if (typeof img.decode === 'function') return img.decode().catch(() => {});+ return new Promise((resolve) => {+ img.addEventListener('load', resolve, { once: true });+ img.addEventListener('error', resolve, { once: true });+ });+ }));+ const media = Array.from(document.querySelectorAll('audio,video'));+ await Promise.all(media.map((m) => {+ if (m.readyState >= 1) return Promise.resolve();+ return new Promise((resolve) => {+ m.addEventListener('loadedmetadata', resolve, { once: true });+ m.addEventListener('error', resolve, { once: true });+ setTimeout(resolve, 2000);+ });+ }));+})()+`, true);+}++async function seek(win: ElectronBrowserWindow, tSec: number): Promise<void> {+ await win.webContents.executeJavaScript(`+(async () => {+ if (window.__ORKAS_VIDEO__ && typeof window.__ORKAS_VIDEO__.seek === 'function') {+ await window.__ORKAS_VIDEO__.seek(${JSON.stringify(tSec)});+ }+})()+`, true);+}++function sampleTimes(durationSec: number): number[] {+ const dur = Math.max(0.1, durationSec);+ return [...new Set([0, dur * 0.25, dur * 0.5, dur * 0.75, Math.max(0, dur - 0.05)].map((n) => round2(n)))];+}++export async function inspectComposition(p: CompositionOptions): Promise<VideoStudioResult> {+ const loaded = await loadCompositionMeta(p.compositionDirAbs);+ if (!loaded.meta) {+ return { ok: true, op: 'composition.inspect', findings: findingsJson(loaded.issues) };+ }+ const issues: Issue[] = [...loaded.issues];+ const samples = sampleTimes(loaded.meta.durationSec);+ try {+ await withCompositionWindow(loaded.meta, p, async (win) => {+ for (const t of samples) {+ await seek(win, t);+ const sampleIssues = await win.webContents.executeJavaScript(buildInspectScript(loaded.meta, t), true) as Issue[];+ issues.push(...sampleIssues);+ }+ });+ } catch (err) {+ issues.push({+ code: 'INSPECT_RENDERER_FAILED',+ severity: 'error',+ selector: 'document',+ message: (err as Error).message,+ source: 'orkas-native',+ });+ }+ return {+ ok: true,+ op: 'composition.inspect',+ findings: findingsJson(issues, {+ engine: 'orkas-native',+ samples,+ canvas: { width: loaded.meta.width, height: loaded.meta.height, durationSec: loaded.meta.durationSec },+ }),+ };+}++function buildInspectScript(meta: CompositionMeta, tSec: number): string {+ return `+(() => {+ const issues = [];+ const width = ${meta.width};+ const height = ${meta.height};+ const tSec = ${tSec};+ const selectorFor = (el) => {+ if (!el || !el.tagName) return 'document';+ if (el.id) return '#' + el.id;+ const cls = typeof el.className === 'string' && el.className.trim() ? '.' + el.className.trim().split(/\\s+/).slice(0, 2).join('.') : '';+ return el.tagName.toLowerCase() + cls;+ };+ const add = (code, severity, el, message) => issues.push({+ code, severity, selector: selectorFor(el), message: '[' + tSec.toFixed(2) + 's] ' + message, source: 'orkas-native-inspect'+ });+ const visible = (el, style) => style.visibility !== 'hidden' && style.display !== 'none' && Number(style.opacity || 1) > 0.01;+ for (const el of Array.from(document.body.querySelectorAll('*'))) {+ const style = getComputedStyle(el);+ if (!visible(el, style)) continue;+ const rect = el.getBoundingClientRect();+ if (!rect || rect.width <= 1 || rect.height <= 1) continue;+ if (rect.left < -1 || rect.top < -1 || rect.right > width + 1 || rect.bottom > height + 1) {+ add('ELEMENT_OUT_OF_CANVAS', 'warning', el, 'element extends outside the declared video canvas.');+ }+ const text = (el.textContent || '').replace(/\\s+/g, ' ').trim();+ if (text) {+ if (el.scrollWidth > el.clientWidth + 2 || el.scrollHeight > el.clientHeight + 2) {+ add('TEXT_OVERFLOW', 'warning', el, 'text content overflows its box.');+ }+ const fs = parseFloat(style.fontSize || '0');+ if (Number.isFinite(fs) && fs > 0 && fs < 18) {+ add('FONT_TOO_SMALL', 'warning', el, 'text is below the 18px legibility floor.');+ }+ }+ }+ return issues;+})()+`;+}++export async function snapshotComposition(p: CompositionOptions): Promise<VideoStudioResult> {+ const loaded = await loadCompositionMeta(p.compositionDirAbs);+ if (!loaded.meta) {+ return { ok: false, op: 'composition.snapshot', errorCode: 'E_COMPOSITION_INVALID', message: loaded.issues[0]?.message || 'composition invalid' };+ }+ if (!p.snapshotAbsPath) {+ return { ok: false, op: 'composition.snapshot', errorCode: 'E_OUTPUT_REQUIRED', message: 'snapshot output path is required.' };+ }+ await fs.mkdir(path.dirname(p.snapshotAbsPath), { recursive: true });+ await withCompositionWindow(loaded.meta, p, async (win) => {+ await seek(win, 0);+ const image = await win.webContents.capturePage({ x: 0, y: 0, width: loaded.meta!.width, height: loaded.meta!.height });+ await fs.writeFile(p.snapshotAbsPath!, image.toPNG());+ });+ const st = await fs.stat(p.snapshotAbsPath);+ return { ok: true, op: 'composition.snapshot', path: p.snapshotAbsPath, bytes: st.size };+}++export async function renderComposition(p: CompositionOptions): Promise<VideoStudioResult> {+ const loaded = await loadCompositionMeta(p.compositionDirAbs);+ if (!loaded.meta) {+ return { ok: false, op: 'composition.render', errorCode: 'E_COMPOSITION_INVALID', message: loaded.issues[0]?.message || 'composition invalid' };+ }+ const blocking = loaded.issues.filter((i) => i.severity === 'error');+ if (blocking.length) {+ return { ok: false, op: 'composition.render', errorCode: 'E_LINT_BLOCKED', message: blocking[0].message, findings: findingsJson(loaded.issues) };+ }+ if (!p.outputAbsPath) {+ return { ok: false, op: 'composition.render', errorCode: 'E_OUTPUT_REQUIRED', message: 'output path is required.' };+ }+ const bins = bundledFfmpegPaths();+ if (!bins.ffmpeg) {+ return { ok: false, op: 'composition.render', errorCode: 'E_FFMPEG_MISSING', message: 'Bundled ffmpeg not found.' };+ }++ const fps = qualityFps(p.quality, p.fps);+ const totalFrames = Math.max(1, Math.ceil(loaded.meta.durationSec * fps));+ const tmp = await fs.mkdtemp(path.join(os.tmpdir(), 'orkas-video-studio-'));+ const framePattern = path.join(tmp, 'frame-%06d.png');+ try {+ p.onProgress?.({ phase: 'composition.render', message: `Capturing ${totalFrames} frames with the native renderer.`, data: { totalFrames, fps } });+ await withCompositionWindow(loaded.meta, p, async (win) => {+ for (let frame = 0; frame < totalFrames; frame += 1) {+ if (p.signal?.aborted) throw new Error('render aborted');+ const t = frame / fps;+ await seek(win, Math.min(t, Math.max(0, loaded.meta!.durationSec - 0.001)));+ const image = await win.webContents.capturePage({ x: 0, y: 0, width: loaded.meta!.width, height: loaded.meta!.height });+ await fs.writeFile(path.join(tmp, `frame-${String(frame + 1).padStart(6, '0')}.png`), image.toPNG());+ if (frame % Math.max(1, Math.floor(fps * 2)) === 0) {+ p.onProgress?.({ phase: 'composition.render.capture', message: `Captured frame ${frame + 1}/${totalFrames}.`, data: { frame: frame + 1, totalFrames } });+ }+ }+ });+ await fs.mkdir(path.dirname(p.outputAbsPath), { recursive: true });+ const encoded = await encodeFrames({+ ffmpeg: bins.ffmpeg,+ framePattern,+ outputAbsPath: p.outputAbsPath,+ fps,+ format: p.format ?? 'mp4',+ quality: p.quality,+ audioTracks: loaded.meta.audioTracks,+ signal: p.signal,+ });+ if (encoded.ok === false) return encoded;+ const st = await fs.stat(p.outputAbsPath);+ return {+ ok: true,+ op: 'composition.render',+ path: p.outputAbsPath,+ bytes: st.size,+ media: `chat-media://local/${p.outputAbsPath}`,+ engine: 'orkas-native',+ fps,+ frames: totalFrames,+ canvas: { width: loaded.meta.width, height: loaded.meta.height, durationSec: loaded.meta.durationSec },+ };+ } catch (err) {+ return { ok: false, op: 'composition.render', errorCode: p.signal?.aborted ? 'E_RENDER_ABORTED' : 'E_RENDER_FAILED', message: (err as Error).message };+ } finally {+ await fs.rm(tmp, { recursive: true, force: true }).catch(() => {});+ }+}++async function encodeFrames(opts: {+ ffmpeg: string;+ framePattern: string;+ outputAbsPath: string;+ fps: number;+ format: RenderFormat;+ quality?: RenderQuality;+ audioTracks: Array<{ absPath: string; startSec: number }>;+ signal?: AbortSignal;+}): Promise<VideoStudioResult> {+ const args = ['-y', '-framerate', String(opts.fps), '-i', opts.framePattern];+ const audio = opts.audioTracks.find((track) => fss.existsSync(track.absPath));+ if (audio) args.push('-i', audio.absPath);+ if (opts.format === 'webm') {+ args.push('-c:v', 'libvpx-vp9', '-pix_fmt', 'yuv420p', '-b:v', '0', '-crf', String(crfForQuality(opts.quality) + 8));+ if (audio) args.push('-c:a', 'libopus', '-shortest');+ } else {+ args.push('-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-preset', 'veryfast', '-crf', String(crfForQuality(opts.quality)), '-movflags', '+faststart');+ if (audio) args.push('-c:a', 'aac', '-shortest');+ }+ args.push(opts.outputAbsPath);+ const r = await runProcess(opts.ffmpeg, args, { signal: opts.signal, timeoutMs: RENDER_TIMEOUT_MS });+ if (r.aborted) return { ok: false, op: 'composition.render', errorCode: 'E_RENDER_ABORTED', message: 'render aborted.' };+ if (r.timedOut) return { ok: false, op: 'composition.render', errorCode: 'E_RENDER_TIMEOUT', message: 'ffmpeg encode timed out.' };+ if (r.code !== 0) {+ log.warn('ffmpeg encode failed', { code: r.code, stderr: r.stderr.slice(-500) });+ return { ok: false, op: 'composition.render', errorCode: 'E_RENDER_ENCODE_FAILED', message: `ffmpeg exited ${r.code}. ${r.stderr.slice(-1200)}` };+ }+ return { ok: true, op: 'composition.render' };+}++async function runProcess(+ bin: string,+ args: string[],+ opts: { signal?: AbortSignal; timeoutMs?: number } = {},+): Promise<{ code: number | null; stdout: string; stderr: string; timedOut: boolean; aborted: boolean }> {+ return new Promise((resolve) => {+ let child: ReturnType<typeof spawn>;+ try {+ child = spawn(bin, args, { ...(opts.signal ? { signal: opts.signal } : {}) });+ } catch (err) {+ resolve({ code: -1, stdout: '', stderr: (err as Error).message, timedOut: false, aborted: false });+ return;+ }+ const out: string[] = [];+ const err: string[] = [];+ let timedOut = false;+ const timer = opts.timeoutMs+ ? setTimeout(() => { timedOut = true; child.kill('SIGKILL'); }, opts.timeoutMs)+ : null;+ child.stdout?.on('data', (c: Buffer) => out.push(c.toString('utf8')));+ child.stderr?.on('data', (c: Buffer) => err.push(c.toString('utf8')));+ child.stdin?.end();+ child.on('error', (e: Error) => {+ if (timer) clearTimeout(timer);+ resolve({ code: -1, stdout: out.join(''), stderr: e.message, timedOut, aborted: !!opts.signal?.aborted });+ });+ child.on('close', (code) => {+ if (timer) clearTimeout(timer);+ resolve({ code, stdout: out.join(''), stderr: err.join(''), timedOut, aborted: !!opts.signal?.aborted });+ });+ });+}++export async function draftComposition(p: CompositionOptions): Promise<VideoStudioResult> {+ const report: Record<string, unknown> = {+ ok: false,+ op: 'composition.draft',+ engine: 'orkas-native',+ composition_dir: p.compositionDirAbs,+ path: p.outputAbsPath || '',+ steps: {},+ };+ const steps = report.steps as Record<string, unknown>;+ const lint = await lintComposition(p);+ steps.lint = lint;+ if (lint.ok === false) return lint;+ const parsedLint = parseFindings(String(lint.findings || ''));+ if ((parsedLint.errorCount || 0) > 0) {+ report.error = { code: 'E_LINT_BLOCKED', message: 'composition lint failed.' };+ await writeReportIfRequested(p.reportAbsPath, report);+ return { ok: false, op: 'composition.draft', errorCode: 'E_LINT_BLOCKED', message: 'composition lint failed.', report };+ }++ const inspect = await inspectComposition(p);+ steps.inspect = inspect;+ if (inspect.ok === false) return inspect;+ if (p.findingsAbsPath && inspect.ok) {+ await fs.mkdir(path.dirname(p.findingsAbsPath), { recursive: true });+ await fs.writeFile(p.findingsAbsPath, String(inspect.findings || ''), 'utf8');+ }++ const render = await renderComposition(p);+ steps.render = render;+ if (render.ok === false) {+ report.error = { code: render.errorCode, message: render.message };+ await writeReportIfRequested(p.reportAbsPath, report);+ return { ...render, report };+ }++ report.ok = true;+ report.media = { path: render.path, bytes: render.bytes };+ await writeReportIfRequested(p.reportAbsPath, report);+ return {+ ok: true,+ op: 'composition.draft',+ path: render.path,+ bytes: render.bytes,+ report_path: p.reportAbsPath || '',+ findings_path: p.findingsAbsPath || '',+ media: `chat-media://local/${render.path}`,+ report,+ };+}++function parseFindings(findings: string): { errorCount?: number } {+ try {+ const parsed = JSON.parse(findings);+ return { errorCount: typeof parsed.errorCount === 'number' ? parsed.errorCount : undefined };+ } catch {+ return {};+ }+}++async function writeReportIfRequested(reportAbsPath: string | undefined, report: Record<string, unknown>): Promise<void> {+ if (!reportAbsPath) return;+ await fs.mkdir(path.dirname(reportAbsPath), { recursive: true });+ await fs.writeFile(reportAbsPath, JSON.stringify(report, null, 2), 'utf8');+ report.report_path = reportAbsPath;+}++function resolveWhisperBackend(): { cli: string; model: string } | null {+ const cli = process.env.ORKAS_WHISPER_CPP || process.env.ORKAS_WHISPER_CLI || '';+ const model = process.env.ORKAS_WHISPER_MODEL || '';+ if (!cli || !model) return null;+ try {+ if (!fss.statSync(cli).isFile() || !fss.statSync(model).isFile()) return null;+ } catch {+ return null;+ }+ return { cli, model };+}++export async function transcribeSpeech(p: SpeechTranscribeOptions): Promise<VideoStudioResult> {+ const backend = resolveWhisperBackend();+ if (!backend) {+ return {+ ok: false,+ op: 'speech.transcribe',+ errorCode: 'E_VIDEO_STUDIO_UNAVAILABLE',+ message: 'Orkas-native speech transcription backend is not installed/configured.',+ };+ }+ const bins = bundledFfmpegPaths();+ if (!bins.ffmpeg) {+ return { ok: false, op: 'speech.transcribe', errorCode: 'E_FFMPEG_MISSING', message: 'Bundled ffmpeg not found.' };+ }+ const st = await fs.stat(p.inputAbsPath).catch(() => null);+ if (!st || !st.isFile()) {+ return { ok: false, op: 'speech.transcribe', errorCode: 'E_TRANSCRIBE_NO_INPUT', message: `input is not a file: ${p.inputAbsPath}` };+ }+ const tmp = await fs.mkdtemp(path.join(os.tmpdir(), 'orkas-transcribe-'));+ const wav = path.join(tmp, 'audio.wav');+ const outBase = path.join(tmp, 'transcript');+ try {+ p.onProgress?.({ phase: 'speech.transcribe.extract', message: 'Extracting mono 16 kHz audio for transcription.' });+ const ex = await runProcess(bins.ffmpeg, ['-y', '-i', p.inputAbsPath, '-vn', '-ac', '1', '-ar', '16000', wav], { signal: p.signal, timeoutMs: 20 * 60 * 1000 });+ if (ex.code !== 0) {+ return { ok: false, op: 'speech.transcribe', errorCode: 'E_TRANSCRIBE_AUDIO_EXTRACT_FAILED', message: ex.stderr.slice(-1200) };+ }+ p.onProgress?.({ phase: 'speech.transcribe.asr', message: 'Running Orkas-native whisper.cpp transcription.' });+ const args = ['-m', backend.model, '-f', wav, '-oj', '-of', outBase];+ if (p.language && p.language !== 'auto') args.push('-l', p.language);+ const tr = await runProcess(backend.cli, args, { signal: p.signal, timeoutMs: 45 * 60 * 1000 });+ if (tr.code !== 0) {+ return { ok: false, op: 'speech.transcribe', errorCode: 'E_TRANSCRIBE_FAILED', message: tr.stderr.slice(-1200) || tr.stdout.slice(-1200) };+ }+ const jsonPath = `${outBase}.json`;+ const raw = await fs.readFile(jsonPath, 'utf8').catch(() => '');+ if (!raw.trim()) {+ return { ok: false, op: 'speech.transcribe', errorCode: 'E_TRANSCRIBE_NO_OUTPUT', message: 'transcriber produced no JSON output.' };+ }+ let parsed: unknown;+ try { parsed = JSON.parse(raw); }+ catch { parsed = { raw }; }+ if (p.transcriptAbsPath) {+ await fs.mkdir(path.dirname(p.transcriptAbsPath), { recursive: true });+ await fs.writeFile(p.transcriptAbsPath, JSON.stringify(parsed, null, 2), 'utf8');+ }+ return {+ ok: true,+ op: 'speech.transcribe',+ summary: parsed,+ transcript_path: p.transcriptAbsPath || '',+ backend: 'orkas-native:whisper.cpp',+ };+ } finally {+ await fs.rm(tmp, { recursive: true, force: true }).catch(() => {});+ }+}diff --git a/src/main/model/core-agent/local-tools.ts b/src/main/model/core-agent/local-tools.tsindex b445de3b..67092676 100644--- a/src/main/model/core-agent/local-tools.ts+++ b/src/main/model/core-agent/local-tools.ts@@ -69,7 +69,13 @@ import { uniquifyPath, renderRenameSignal } from '../../util/uniquify-path';import { isPathAllowed } from '../../util/path-sandbox';import { kindOf } from '../../features/file_indexer';import { getWorkspacePath } from '../../features/user_workspace';-import { chatAttachmentDir, userMarketplaceSkillsDir, userSkillsDir } from '../../paths';+import {+ chatAttachmentDir,+ userMarketplaceAgentsDir,+ userMarketplaceSkillsDir,+ userSystemSkillsDir,+ userSkillsDir,+} from '../../paths';import * as chatArtifacts from '../../features/chat_artifacts';import { finalizeProducedArtifact, producedDocumentFooterText } from '../../features/produced_output_hooks';import { readDisabledSets } from '../../features/component_enabled';@@ -126,6 +132,10 @@ export interface LocalToolsOpts {* on top of workspace + attachment. Used by skill-edit / agent-edit* chats so the LLM can edit files inside the skill / agent dir. */extraRoots?: readonly string[];+ /** Extra roots that read tools may expose but local execution must never+ * mutate. This is a deny-only lane: it does not make paths readable or+ * writable for localTools, and it overrides all-files access modes. */+ readOnlyExtraRoots?: readonly string[];/** Fires with absolute path after every successful write (write_file,* edit_file, markdown_to_pdf, html_to_pdf). Lets chats.ts surface* produced files to the UI. */@@ -372,6 +382,57 @@ function _sameOrInside(parent: string, candidate: string): boolean {return rel === '' || (!!rel && !rel.startsWith('..') && !path.isAbsolute(rel));}+function _uniqueResolvedRoots(input: readonly string[]): string[] {+ const out: string[] = [];+ const seen = new Set<string>();+ for (const root of input) {+ if (!root) continue;+ const abs = path.resolve(root);+ if (seen.has(abs)) continue;+ seen.add(abs);+ out.push(abs);+ }+ return out;+}++function protectedWriteRootsFor(opts: LocalToolsOpts): string[] {+ const roots: string[] = [];+ if (opts.userId) {+ roots.push(+ userMarketplaceAgentsDir(opts.userId),+ userMarketplaceSkillsDir(opts.userId),+ userSystemSkillsDir(opts.userId),+ );+ }+ if (opts.readOnlyExtraRoots?.length) roots.push(...opts.readOnlyExtraRoots);+ return _uniqueResolvedRoots(roots);+}++function protectedRootForPath(opts: LocalToolsOpts, abs: string): string | null {+ const candidate = path.resolve(abs);+ for (const root of protectedWriteRootsFor(opts)) {+ if (_sameOrInside(root, candidate)) return root;+ }+ return null;+}++function protectedRootMentionedByCommand(opts: LocalToolsOpts, command: string): string | null {+ if (!command) return null;+ const normalized = command.replace(/\\([\\ "'$`])/g, '$1').replace(/\\/g, '/');+ for (const root of protectedWriteRootsFor(opts)) {+ const resolved = path.resolve(root).replace(/\\/g, '/');+ if (normalized.includes(resolved)) return root;+ }+ return null;+}++function protectedWriteError(abs: string, root: string): string {+ return errText(+ 'E_PROTECTED_PATH_READ_ONLY',+ `path is inside a protected read-only Orkas resource root and cannot be modified by local tools: ${abs} (root: ${root}). Use the agent/skill edit or fork flow instead.`,+ );+}+function _addDownloadDir(out: string[], root: string, target: string): void {if (!target || /^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(target)) return;const abs = path.resolve(root, target);@@ -640,6 +701,50 @@ function parseOrkasCliInvocation(};}+/** Minimum interval between forwarded script progress events; the 60s+ * heartbeat still carries the latest one in between. */+const SCRIPT_PROGRESS_FORWARD_MIN_MS = 5_000;++/**+ * Incremental scanner for the skill-script progress protocol: scripts (video+ * skills et al) write `{"type":"progress",...}` JSONL to stderr. feed() returns+ * the parsed payloads of any complete progress lines in the chunk; other+ * stderr content is ignored.+ */+export function createScriptProgressScanner(): { feed(chunk: string): Array<Record<string, unknown>> } {+ let buf = '';+ return {+ feed(chunk: string) {+ buf += chunk;+ const events: Array<Record<string, unknown>> = [];+ let nl: number;+ while ((nl = buf.indexOf('\n')) !== -1) {+ const line = buf.slice(0, nl).trim();+ buf = buf.slice(nl + 1);+ if (!line.startsWith('{') || !line.includes('"progress"')) continue;+ try {+ const parsed = JSON.parse(line) as Record<string, unknown>;+ if (parsed && parsed.type === 'progress') events.push(parsed);+ } catch { /* not a progress line */ }+ }+ // A stream that never emits a newline must not grow the buffer forever.+ if (buf.length > 64 * 1024) buf = buf.slice(-8 * 1024);+ return events;+ },+ };+}++/** One-line human summary of a script progress payload for the tool progress UI. */+export function formatScriptProgress(p: Record<string, unknown>): string {+ const s = (v: unknown) => (typeof v === 'string' && v ? v : '');+ if (s(p.message)) return s(p.message);+ const parts = [s(p.source), s(p.op), s(p.phase), s(p.status)].filter((v, i, a) => v && a.indexOf(v) === i);+ const pct = typeof p.percent === 'number' && Number.isFinite(p.percent) ? ` ${Math.round(p.percent as number)}%` : '';+ const t = typeof p.out_time_sec === 'number' && Number.isFinite(p.out_time_sec) ? ` t=${Math.round(p.out_time_sec as number)}s` : '';+ const elapsed = typeof p.elapsed_sec === 'number' && Number.isFinite(p.elapsed_sec) ? ` (${Math.round(p.elapsed_sec as number)}s elapsed)` : '';+ return parts.length ? `${parts.join(' ')}${pct}${t}${elapsed}` : 'script progress';+}+async function executeDirectOrkasCli(invocation: OrkasCliInvocation,input: Record<string, unknown>,@@ -731,7 +836,24 @@ async function executeDirectOrkasCli(};child.stdout.on('data', (data: Buffer) => append('stdout', data));- child.stderr.on('data', (data: Buffer) => append('stderr', data));+ const progressScanner = createScriptProgressScanner();+ let lastScriptProgress: Record<string, unknown> | null = null;+ let lastProgressForwardAt = 0;+ child.stderr.on('data', (data: Buffer) => {+ append('stderr', data);+ if (!ctx.emitProgress) return;+ for (const ev of progressScanner.feed(data.toString('utf8'))) {+ lastScriptProgress = ev;+ const now = Date.now();+ if (now - lastProgressForwardAt < SCRIPT_PROGRESS_FORWARD_MIN_MS) continue;+ lastProgressForwardAt = now;+ ctx.emitProgress({+ phase: 'running',+ message: formatScriptProgress(ev),+ data: { script_progress: ev },+ });+ }+ });const timeout = setTimeout(() => {timedOut = true;@@ -748,10 +870,16 @@ async function executeDirectOrkasCli(if (ctx.emitProgress) {heartbeat = setInterval(() => {const elapsedMs = Date.now() - startedAt;+ const scriptNote = lastScriptProgress ? `; last: ${formatScriptProgress(lastScriptProgress)}` : '';ctx.emitProgress?.({phase: 'running',- message: `Command still running (${formatBashDuration(elapsedMs)} elapsed; timeout ${formatBashDuration(timeoutMs)})`,- data: { elapsedMs, timeoutMs, heartbeat: true },+ message: `Command still running (${formatBashDuration(elapsedMs)} elapsed; timeout ${formatBashDuration(timeoutMs)})${scriptNote}`,+ data: {+ elapsedMs,+ timeoutMs,+ heartbeat: true,+ ...(lastScriptProgress ? { script_progress: lastScriptProgress } : {}),+ },});}, BASH_PROGRESS_INTERVAL_MS);if (typeof heartbeat.unref === 'function') heartbeat.unref();@@ -826,6 +954,8 @@ function allowedRootsFor(opts: LocalToolsOpts): string[] {}function guardEditPath(opts: LocalToolsOpts, abs: string): string | null {+ const protectedRoot = protectedRootForPath(opts, abs);+ if (protectedRoot) return protectedWriteError(abs, protectedRoot);const roots = allowedRootsFor(opts);if (!roots.length && !localAccessAllowsOutsideWorkspace()) {return errText('E_NO_SCOPE', 'no visible roots for this conversation');@@ -891,6 +1021,8 @@ async function gateEditPath(opts: LocalToolsOpts, abs: string, ctx?: ToolContext}function guardDeletePath(opts: LocalToolsOpts, abs: string): string | null {+ const protectedRoot = protectedRootForPath(opts, abs);+ if (protectedRoot) return protectedWriteError(abs, protectedRoot);const roots = allowedRootsFor(opts);if (!roots.length && !localAccessAllowsOutsideWorkspace()) {return errText('E_NO_SCOPE', 'no visible roots for this conversation');@@ -1339,6 +1471,10 @@ function bashWritableRootsFor(opts: LocalToolsOpts, workingDir: string): string[}function guardBashScopedPath(opts: LocalToolsOpts, abs: string, workingDir: string, access: 'read' | 'write'): string | null {+ if (access === 'write') {+ const protectedRoot = protectedRootForPath(opts, abs);+ if (protectedRoot) return protectedWriteError(abs, protectedRoot);+ }const roots = bashScopedRootsFor(opts, workingDir);if (!roots.length) {return localAccessAllowsOutsideWorkspace()@@ -1652,6 +1788,18 @@ function createBashTool(opts: LocalToolsOpts): AgentTool {async execute(input, ctx) {const mode = getLocalExecMode();const command = String(input.command ?? '');+ const protectedMention = protectedRootMentionedByCommand(opts, command);+ if (protectedMention) {+ log.warn('bash protected root reject', {+ user_id: maskId(opts.userId),+ command_chars: command.length,+ root: logPathRef(protectedMention),+ });+ return {+ content: protectedWriteError(protectedMention, protectedMention),+ isError: true,+ };+ }const oauthClientMismatchErr = guardGoogleWorkspaceOauthClientMismatchCommand(command, ctx.workingDir);if (oauthClientMismatchErr) {log.warn('bash google oauth client/scope mismatch reject', {@@ -1766,6 +1914,18 @@ async function gateInteractiveCliStart(ctx?: ToolContext,): Promise<ToolResult | null> {const mode = getLocalExecMode();+ const protectedMention = protectedRootMentionedByCommand(opts, command);+ if (protectedMention) {+ log.warn('interactive_cli_start protected root reject', {+ user_id: maskId(opts.userId),+ command_chars: command.length,+ root: logPathRef(protectedMention),+ });+ return {+ content: protectedWriteError(protectedMention, protectedMention),+ isError: true,+ };+ }const oauthClientMismatchErr = guardGoogleWorkspaceOauthClientMismatchCommand(command, settings?.workingDir);if (oauthClientMismatchErr) {log.warn('interactive_cli_start google oauth client/scope mismatch reject', {diff --git a/src/main/model/core-agent/office-batch.ts b/src/main/model/core-agent/office-batch.tsindex bb60d1ac..c27f6193 100644--- a/src/main/model/core-agent/office-batch.ts+++ b/src/main/model/core-agent/office-batch.ts@@ -5,7 +5,7 @@* and piped to `officecli batch <file>` over stdin (one open/save cycle).** Op shapes are exactly what `officecli` consumes (verified via `dump` and live- * runs against v1.0.117):+ * runs against v1.0.131):* - docx / pptx body content → `add` ops: {command,parent,type,props}* - xlsx cells → `set` ops: {command,path,props}*/diff --git a/src/main/model/core-agent/tool-catalog.ts b/src/main/model/core-agent/tool-catalog.tsindex 1d9a7e69..97b4b7f6 100644--- a/src/main/model/core-agent/tool-catalog.ts+++ b/src/main/model/core-agent/tool-catalog.ts@@ -83,12 +83,14 @@ export const DEEP_RESEARCH_AGENT_IDS = ['7083ff63b398', // BrandResearcher];+export const VIDEO_STUDIO_AGENT_ID = '79df9cc89f5f';+export const TOOL_CATALOG: ToolCatalogEntry[] = [// Files / workspace{ name: 'read_file', group: 'fs', summary: 'Read a slice of text from a workspace or attachment file (PDF/modern Office text or image as multimodal).' },{ name: 'write_file', group: 'fs', permission: 'localExec', summary: 'Write text/code/markdown into the workspace; resolves under $working_dir.' },{ name: 'edit_file', group: 'fs', permission: 'localExec', summary: 'In-place `old_string → new_string` replacement on an existing text file (instead of rewriting the whole file).' },- { name: 'delete_file', group: 'fs', permission: 'localExec', summary: 'Delete a single file from the workspace / current attachment dir / extraRoots. The first call shows an inline confirmation card and returns a token; multiple deletes from the same turn are grouped in one card when possible. Use instead of `bash rm` for removals.' },+ { name: 'delete_file', group: 'fs', permission: 'localExec', summary: 'Delete a single file. Files inside the current workspace/attachment/editor scope are deleted immediately; files outside that scope use an inline confirmation card with a token, and multiple out-of-scope deletes from the same turn are grouped when possible. Use instead of `bash rm` for removals.' },{ name: 'list_files', group: 'fs', summary: 'List the workspace directory tree.' },{ name: 'stat_file', group: 'fs', summary: 'Trigger PDF/modern Office extraction and return total_chars; call before read_file.' },{ name: 'ocr_file', group: 'fs', summary: 'Run local OCR on PDF pages or image files when visual text is not available through read_file/stat_file.' },@@ -127,6 +129,7 @@ export const TOOL_CATALOG: ToolCatalogEntry[] = [// Image{ name: 'generate_image', group: 'image', permission: 'localExec', summary: 'Call the configured image-generation API and save the result into the workspace.' },+ { name: 'video_studio', group: 'video', permission: 'localExec', ownerAgent: VIDEO_STUDIO_AGENT_ID, summary: 'VideoStudio-owned native runtime for HyperFrames-compatible HTML composition lint/inspect/render and speech transcription fallback orchestration.' },// Web (when a vendor-native search is available the framework picks it automatically; the two below are the fallback channel){ name: 'web_search', group: 'web', summary: 'Built-in fallback web search (vendor-native search is preferred automatically when available).' },diff --git a/src/main/model/core-agent/video-studio-tool.ts b/src/main/model/core-agent/video-studio-tool.tsnew file mode 100644index 00000000..6dd293d1--- /dev/null+++ b/src/main/model/core-agent/video-studio-tool.ts@@ -0,0 +1,288 @@+/**+ * VideoStudio-owned native runtime tool.+ *+ * This intentionally covers only the VideoStudio dependency points that need+ * to be native:+ * HTML composition render/lint/inspect and speech transcription. The rest of+ * VideoStudio's agent-private scripts stay script-owned.+ */++import * as fs from 'node:fs/promises';+import * as path from 'node:path';++import type { AgentTool, ToolContext, ToolResult } from '#core-agent';+import { getLocalExecGranted } from '../../features/permissions';+import {+ draftComposition,+ inspectComposition,+ lintComposition,+ renderComposition,+ snapshotComposition,+ transcribeSpeech,+ type RenderFormat,+ type RenderQuality,+ type VideoStudioOp,+} from '../../features/video_studio';+import { isPathAllowed } from '../../util/path-sandbox';+import { uniquifyPath, renderRenameSignal } from '../../util/uniquify-path';+import { getWorkspacePath } from '../../features/user_workspace';+import { chatAttachmentDir } from '../../paths';+import { createLogger } from '../../logger';++const log = createLogger('video-studio-tool');++const DENY_MESSAGE =+ 'E_TOOL_EXECUTION_ACCESS_DISABLED: Tool execution access is disabled, so VideoStudio native rendering/transcription was not run.';++export interface VideoStudioToolOpts {+ userId: string;+ cid?: string;+ projectId?: string;+ extraRoots?: readonly string[];+ onFileWritten?: (absPath: string) => void | Promise<void>;+ hasProducedPath?: (absPath: string) => boolean;+}++const OPS = new Set<VideoStudioOp>([+ 'composition.lint',+ 'composition.inspect',+ 'composition.render',+ 'composition.draft',+ 'composition.snapshot',+ 'speech.transcribe',+]);++function allowedRoots(opts: VideoStudioToolOpts): string[] {+ const roots: string[] = [];+ const push = (value: string | undefined) => {+ if (!value) return;+ const resolved = path.resolve(value);+ if (!roots.includes(resolved)) roots.push(resolved);+ };+ try { push(getWorkspacePath(opts.userId)); }+ catch (err) { log.warn(`resolve workspace failed: ${(err as Error).message}`); }+ if (opts.projectId) {+ try { push(getWorkspacePath(opts.userId, opts.projectId)); }+ catch (err) { log.warn(`resolve project workspace failed: ${(err as Error).message}`); }+ }+ if (opts.cid) {+ try { push(chatAttachmentDir(opts.userId, opts.cid)); }+ catch (err) { log.warn(`resolve attachment dir failed: ${(err as Error).message}`); }+ }+ for (const root of opts.extraRoots || []) push(root);+ return roots;+}++function isProjectRelativePath(p: string): boolean {+ const first = p.replace(/\\/g, '/').split('/').find(Boolean);+ return first === 'project';+}++function defaultRoot(opts: VideoStudioToolOpts, ctx: ToolContext): string {+ if (ctx.workingDir) return ctx.workingDir;+ try { return getWorkspacePath(opts.userId, opts.projectId); }+ catch { return '.'; }+}++function resolvePath(ctx: ToolContext, opts: VideoStudioToolOpts, raw: string, roots: string[]): string {+ const value = String(raw || '').trim();+ if (path.isAbsolute(value)) return path.resolve(value);+ if (ctx.workingDir && isProjectRelativePath(value)) {+ const candidate = path.resolve(ctx.workingDir, value);+ if (isPathAllowed(candidate, roots)) return candidate;+ }+ return path.resolve(defaultRoot(opts, ctx), value);+}++function withExtension(absPath: string, ext: string): string {+ return path.extname(absPath) ? absPath : `${absPath}.${ext.replace(/^\./, '')}`;+}++async function ensureInputFile(absPath: string): Promise<string | null> {+ const st = await fs.stat(absPath).catch(() => null);+ return st && st.isFile() ? null : `input is not a file: ${absPath}`;+}++async function ensureInputDir(absPath: string): Promise<string | null> {+ const st = await fs.stat(absPath).catch(() => null);+ return st && st.isDirectory() ? null : `composition_dir is not a directory: ${absPath}`;+}++async function notifyWritten(opts: VideoStudioToolOpts, paths: Array<unknown>): Promise<void> {+ if (!opts.onFileWritten) return;+ const seen = new Set<string>();+ for (const value of paths) {+ if (typeof value !== 'string' || !value) continue;+ const abs = path.resolve(value);+ if (seen.has(abs)) continue;+ seen.add(abs);+ try { await opts.onFileWritten(abs); }+ catch (err) { log.warn(`onFileWritten failed: ${(err as Error).message}`); }+ }+}++function resultContent(result: Record<string, unknown>, renamedNote = ''): string {+ return `${JSON.stringify(result, null, 2)}${renamedNote}`;+}++export function createVideoStudioTool(opts: VideoStudioToolOpts): AgentTool {+ return {+ name: 'video_studio',+ description:+ 'VideoStudio-native runtime for HTML video compositions and speech transcription. Use composition.* for Orkas-native composition lint/inspect/draft/render/snapshot, and speech.transcribe for Orkas-native transcription. Marketplace min_app_version enforces app compatibility before this agent is installed.',+ inputSchema: {+ type: 'object',+ properties: {+ op: {+ type: 'string',+ enum: [...OPS],+ description: 'Operation: composition.lint, composition.inspect, composition.render, composition.draft, composition.snapshot, or speech.transcribe.',+ },+ composition_dir: { type: 'string', description: 'Directory containing index.html for composition.* ops.' },+ output_path: { type: 'string', description: 'Output video path for composition.render/draft, or snapshot path for composition.snapshot.' },+ report_path: { type: 'string', description: 'Optional JSON report path for composition.draft.' },+ findings_path: { type: 'string', description: 'Optional findings JSON path for composition.inspect/draft.' },+ quality: { type: 'string', enum: ['draft', 'standard', 'high'], description: 'Render quality; draft uses lower fps/CRF.' },+ fps: { type: 'number', description: 'Frames per second, capped at 60.' },+ format: { type: 'string', enum: ['mp4', 'webm'], description: 'Output video format. Default mp4.' },+ variables: { type: 'object', description: 'Optional composition variables exposed as window.__ORKAS_VIDEO_VARIABLES__.' },+ input_path: { type: 'string', description: 'Input audio/video path for speech.transcribe.' },+ transcript_path: { type: 'string', description: 'Optional transcript JSON output path for speech.transcribe.' },+ model: { type: 'string', description: 'ASR model id/path. Backend-specific.' },+ language: { type: 'string', description: 'ASR language code, or auto.' },+ timestamps: { type: 'string', enum: ['segment', 'word'], description: 'ASR timestamp detail.' },+ allow_model_download: { type: 'boolean', description: 'Whether native ASR may download a missing model. Backend-specific.' },+ },+ required: ['op'],+ },+ async execute(input, ctx) {+ if (!getLocalExecGranted()) {+ return { content: DENY_MESSAGE, isError: true } as ToolResult;+ }++ const op = String(input.op || '').trim() as VideoStudioOp;+ if (!OPS.has(op)) {+ return { content: `op must be one of: ${[...OPS].join(', ')}`, isError: true } as ToolResult;+ }++ const roots = allowedRoots(opts);++ if (op.startsWith('composition.')) {+ const compositionRaw = String(input.composition_dir || '').trim();+ if (!compositionRaw) return { content: 'composition_dir is required', isError: true } as ToolResult;+ const compositionDirAbs = resolvePath(ctx, opts, compositionRaw, roots);+ if (!isPathAllowed(compositionDirAbs, roots)) {+ return { content: `E_PATH_OUT_OF_SCOPE: composition_dir is outside scope: ${compositionDirAbs}`, isError: true } as ToolResult;+ }+ const dirErr = await ensureInputDir(compositionDirAbs);+ if (dirErr) return { content: dirErr, isError: true } as ToolResult;++ const format = input.format === 'webm' ? 'webm' as RenderFormat : 'mp4' as RenderFormat;+ const quality = (input.quality === 'standard' || input.quality === 'high' || input.quality === 'draft')+ ? input.quality as RenderQuality+ : undefined;+ const fps = typeof input.fps === 'number' ? input.fps : undefined;+ const variables = input.variables && typeof input.variables === 'object' && !Array.isArray(input.variables)+ ? input.variables as Record<string, unknown>+ : undefined;++ let outputAbsPath: string | undefined;+ let requestedOutput = '';+ let renamed = false;+ if (op === 'composition.render' || op === 'composition.draft') {+ const outputRaw = String(input.output_path || '').trim();+ if (!outputRaw) return { content: 'output_path is required', isError: true } as ToolResult;+ requestedOutput = withExtension(resolvePath(ctx, opts, outputRaw, roots), format);+ if (!isPathAllowed(requestedOutput, roots)) {+ return { content: `E_PATH_OUT_OF_SCOPE: output_path is outside scope: ${requestedOutput}`, isError: true } as ToolResult;+ }+ const isMine = opts.hasProducedPath ? (p: string) => opts.hasProducedPath!(p) : () => false;+ const unique = await uniquifyPath(requestedOutput, isMine);+ outputAbsPath = unique.finalPath;+ renamed = unique.renamed;+ } else if (op === 'composition.snapshot') {+ const outputRaw = String(input.output_path || '').trim();+ if (!outputRaw) return { content: 'output_path is required for composition.snapshot', isError: true } as ToolResult;+ outputAbsPath = withExtension(resolvePath(ctx, opts, outputRaw, roots), 'png');+ if (!isPathAllowed(outputAbsPath, roots)) {+ return { content: `E_PATH_OUT_OF_SCOPE: output_path is outside scope: ${outputAbsPath}`, isError: true } as ToolResult;+ }+ }++ const reportAbsPath = typeof input.report_path === 'string' && input.report_path.trim()+ ? resolvePath(ctx, opts, input.report_path, roots)+ : undefined;+ if (reportAbsPath && !isPathAllowed(reportAbsPath, roots)) {+ return { content: `E_PATH_OUT_OF_SCOPE: report_path is outside scope: ${reportAbsPath}`, isError: true } as ToolResult;+ }+ const findingsAbsPath = typeof input.findings_path === 'string' && input.findings_path.trim()+ ? resolvePath(ctx, opts, input.findings_path, roots)+ : undefined;+ if (findingsAbsPath && !isPathAllowed(findingsAbsPath, roots)) {+ return { content: `E_PATH_OUT_OF_SCOPE: findings_path is outside scope: ${findingsAbsPath}`, isError: true } as ToolResult;+ }++ const common = {+ compositionDirAbs,+ ...(outputAbsPath && op !== 'composition.snapshot' ? { outputAbsPath } : {}),+ ...(outputAbsPath && op === 'composition.snapshot' ? { snapshotAbsPath: outputAbsPath } : {}),+ ...(reportAbsPath ? { reportAbsPath } : {}),+ ...(findingsAbsPath ? { findingsAbsPath } : {}),+ ...(quality ? { quality } : {}),+ ...(typeof fps === 'number' ? { fps } : {}),+ format,+ ...(variables ? { variables } : {}),+ ...(ctx.signal ? { signal: ctx.signal } : {}),+ onProgress: (event: { phase: string; message: string; data?: Record<string, unknown> }) => ctx.emitProgress?.(event),+ };+ const result = op === 'composition.lint'+ ? await lintComposition(common)+ : op === 'composition.inspect'+ ? await inspectComposition(common)+ : op === 'composition.snapshot'+ ? await snapshotComposition(common)+ : op === 'composition.draft'+ ? await draftComposition(common)+ : await renderComposition(common);++ if (result.ok) {+ await notifyWritten(opts, [+ result.path,+ result.report_path,+ result.findings_path,+ ]);+ }+ const renameNote = renamed && outputAbsPath ? renderRenameSignal(requestedOutput, outputAbsPath) : '';+ return { content: resultContent(result, renameNote), isError: result.ok === false } as ToolResult;+ }++ const inputRaw = String(input.input_path || '').trim();+ if (!inputRaw) return { content: 'input_path is required for speech.transcribe', isError: true } as ToolResult;+ const inputAbsPath = resolvePath(ctx, opts, inputRaw, roots);+ if (!isPathAllowed(inputAbsPath, roots)) {+ return { content: `E_PATH_OUT_OF_SCOPE: input_path is outside scope: ${inputAbsPath}`, isError: true } as ToolResult;+ }+ const fileErr = await ensureInputFile(inputAbsPath);+ if (fileErr) return { content: fileErr, isError: true } as ToolResult;++ const transcriptAbsPath = typeof input.transcript_path === 'string' && input.transcript_path.trim()+ ? resolvePath(ctx, opts, input.transcript_path, roots)+ : undefined;+ if (transcriptAbsPath && !isPathAllowed(transcriptAbsPath, roots)) {+ return { content: `E_PATH_OUT_OF_SCOPE: transcript_path is outside scope: ${transcriptAbsPath}`, isError: true } as ToolResult;+ }+ const result = await transcribeSpeech({+ inputAbsPath,+ ...(transcriptAbsPath ? { transcriptAbsPath } : {}),+ ...(typeof input.model === 'string' && input.model.trim() ? { model: input.model.trim() } : {}),+ ...(typeof input.language === 'string' && input.language.trim() ? { language: input.language.trim() } : {}),+ timestamps: input.timestamps === 'segment' ? 'segment' : 'word',+ allowModelDownload: input.allow_model_download === true,+ ...(ctx.signal ? { signal: ctx.signal } : {}),+ onProgress: (event) => ctx.emitProgress?.(event),+ });+ if (result.ok) await notifyWritten(opts, [result.transcript_path]);+ return { content: resultContent(result), isError: result.ok === false } as ToolResult;+ },+ };+}diff --git a/test/main/model/core-agent/local-tools.test.ts b/test/main/model/core-agent/local-tools.test.tsindex 46e76720..016888c8 100644--- a/test/main/model/core-agent/local-tools.test.ts+++ b/test/main/model/core-agent/local-tools.test.ts@@ -25,7 +25,7 @@ afterEach(() => {fs.rmSync(tmpDir, { recursive: true, force: true });});-async function buildEditTool(opts: { onFileWritten?: (p: string) => void; extraRoots?: string[] } = {}) {+async function buildEditTool(opts: { onFileWritten?: (p: string) => void; extraRoots?: string[]; readOnlyExtraRoots?: string[] } = {}) {const localTools = await import('../../../../src/main/model/core-agent/local-tools');const ws = await import('../../../../src/main/features/user_workspace');const wsDir = path.join(tmpDir, 'ws');@@ -37,13 +37,14 @@ async function buildEditTool(opts: { onFileWritten?: (p: string) => void; extraRcid: CID,...(opts.onFileWritten ? { onFileWritten: opts.onFileWritten } : {}),...(opts.extraRoots ? { extraRoots: opts.extraRoots } : {}),+ ...(opts.readOnlyExtraRoots ? { readOnlyExtraRoots: opts.readOnlyExtraRoots } : {}),});const edit = tools.find((t) => t.name === 'edit_file');if (!edit) throw new Error('edit_file tool missing');return { edit, wsDir };}-async function buildWriteTool(opts: { onFileWritten?: (p: string) => void; extraRoots?: string[] } = {}) {+async function buildWriteTool(opts: { onFileWritten?: (p: string) => void; extraRoots?: string[]; readOnlyExtraRoots?: string[] } = {}) {const localTools = await import('../../../../src/main/model/core-agent/local-tools');const ws = await import('../../../../src/main/features/user_workspace');const wsDir = path.join(tmpDir, 'ws');@@ -55,6 +56,7 @@ async function buildWriteTool(opts: { onFileWritten?: (p: string) => void; extracid: CID,...(opts.onFileWritten ? { onFileWritten: opts.onFileWritten } : {}),...(opts.extraRoots ? { extraRoots: opts.extraRoots } : {}),+ ...(opts.readOnlyExtraRoots ? { readOnlyExtraRoots: opts.readOnlyExtraRoots } : {}),});const write = tools.find((t) => t.name === 'write_file');if (!write) throw new Error('write_file tool missing');@@ -202,8 +204,8 @@ describe('local-tools › edit_file › sandbox', () => {expect(fs.existsSync(outside)).toBe(false);});- it('does not allow delete_file under readOnlyExtraRoots', async () => {- await workspaceOnly();+ it('does not allow delete_file under readOnlyExtraRoots, even in all-files mode', async () => {+ await allFilesApproval();const readOnlyDir = fs.mkdtempSync(path.join(os.tmpdir(), 'orkas-ro-delete-'));const p = path.join(readOnlyDir, 'keep.md');fs.writeFileSync(p, 'do not delete');@@ -211,12 +213,50 @@ describe('local-tools › edit_file › sandbox', () => {const { del } = await buildDeleteTool({ readOnlyExtraRoots: [readOnlyDir] });const r = await run(del, { path: p });expect(r.isError).toBe(true);- expect(r.content).toContain('E_PATH_OUT_OF_SCOPE');+ expect(r.content).toContain('E_PROTECTED_PATH_READ_ONLY');expect(fs.readFileSync(p, 'utf8')).toBe('do not delete');} finally {fs.rmSync(readOnlyDir, { recursive: true, force: true });}});++ it('blocks direct local-tool mutation of marketplace installs in all-files mode', async () => {+ await allFilesApproval();+ const paths = await import('../../../../src/main/paths');+ const localTools = await import('../../../../src/main/model/core-agent/local-tools');+ const skillDir = paths.userMarketplaceSkillDir(UID, 'platform-skill');+ fs.mkdirSync(skillDir, { recursive: true });+ const skillFile = path.join(skillDir, 'SKILL.md');+ fs.writeFileSync(skillFile, 'original skill');++ const tools = localTools.createLocalTools({ userId: UID, cid: CID });+ const edit = tools.find((t) => t.name === 'edit_file');+ const write = tools.find((t) => t.name === 'write_file');+ const del = tools.find((t) => t.name === 'delete_file');+ const bash = tools.find((t) => t.name === 'bash');+ if (!edit || !write || !del || !bash) throw new Error('expected local tools missing');++ const editRes = await run(edit, { path: skillFile, old_string: 'original', new_string: 'mutated' });+ expect(editRes.isError).toBe(true);+ expect(editRes.content).toContain('E_PROTECTED_PATH_READ_ONLY');+ expect(fs.readFileSync(skillFile, 'utf8')).toBe('original skill');++ const newFile = path.join(skillDir, 'notes.md');+ const writeRes = await run(write, { path: newFile, content: 'new bytes' });+ expect(writeRes.isError).toBe(true);+ expect(writeRes.content).toContain('E_PROTECTED_PATH_READ_ONLY');+ expect(fs.existsSync(newFile)).toBe(false);++ const deleteRes = await run(del, { path: skillFile });+ expect(deleteRes.isError).toBe(true);+ expect(deleteRes.content).toContain('E_PROTECTED_PATH_READ_ONLY');+ expect(fs.existsSync(skillFile)).toBe(true);++ const bashRes = await run(bash, { command: `printf hacked > "${skillFile}"` });+ expect(bashRes.isError).toBe(true);+ expect(bashRes.content).toContain('E_PROTECTED_PATH_READ_ONLY');+ expect(fs.readFileSync(skillFile, 'utf8')).toBe('original skill');+ });});describe('local-tools › delete_file › confirmation scope', () => {@@ -587,3 +627,48 @@ describe('local-tools › create_artifact › backend rejection surfaces as isErexpect(created).toEqual([]);});});++describe('local-tools › direct CLI › script progress scanner', () => {+ async function scannerModule() {+ return import('../../../../src/main/model/core-agent/local-tools');+ }++ it('parses progress JSONL lines, including across chunk boundaries', async () => {+ const { createScriptProgressScanner } = await scannerModule();+ const scanner = createScriptProgressScanner();+ const first = scanner.feed('{"type":"progress","source":"video_edit","op":"trim","status":"encoding"}\n{"type":"prog');+ expect(first).toHaveLength(1);+ expect(first[0]).toMatchObject({ source: 'video_edit', op: 'trim', status: 'encoding' });+ // Second half of the split line arrives in the next chunk.+ const second = scanner.feed('ress","source":"video_analyze","op":"transcribe","status":"heartbeat"}\n');+ expect(second).toHaveLength(1);+ expect(second[0]).toMatchObject({ source: 'video_analyze', op: 'transcribe' });+ });++ it('ignores non-progress stderr content and look-alike lines', async () => {+ const { createScriptProgressScanner } = await scannerModule();+ const scanner = createScriptProgressScanner();+ const events = scanner.feed([+ 'ffmpeg version 6.0 stderr noise',+ '{"ok":false,"code":"E_EDIT_FAILED","message":"boom"}',+ '{"type":"progressive","source":"look-alike"}',+ '{"type":"progress" broken json',+ '"type":"progress" not an object line',+ '{"type":"progress","source":"video_edit","status":"heartbeat"}',+ '',+ ].join('\n') + '\n');+ expect(events).toHaveLength(1);+ expect(events[0]).toMatchObject({ type: 'progress', source: 'video_edit' });+ });++ it('formats a readable one-liner and prefers an explicit message', async () => {+ const { formatScriptProgress } = await scannerModule();+ expect(formatScriptProgress({ message: 'rendering scene 2/4' })).toBe('rendering scene 2/4');+ const line = formatScriptProgress({ source: 'video_edit', op: 'trim', status: 'encoding', percent: 42.4, out_time_sec: 12.6 });+ expect(line).toContain('video_edit');+ expect(line).toContain('trim');+ expect(line).toContain('42%');+ expect(line).toContain('t=13s');+ expect(formatScriptProgress({})).toBe('script progress');+ });+});diff --git a/test/main/model/core-agent/tool-catalog.test.ts b/test/main/model/core-agent/tool-catalog.test.tsindex c3fb2d58..97889390 100644--- a/test/main/model/core-agent/tool-catalog.test.ts+++ b/test/main/model/core-agent/tool-catalog.test.ts@@ -289,6 +289,13 @@ describe('tool-catalog', () => {expect(bashLine).toContain('local-execution permission');expect(readLine).not.toContain('local-execution permission');});++ it('keeps delete_file confirmation guidance scoped to outside-workspace deletes', () => {+ const out = getToolsSystemPromptBlock(['delete_file']);+ expect(out).toContain('Files inside the current workspace/attachment/editor scope are deleted immediately');+ expect(out).toContain('files outside that scope use an inline confirmation card');+ expect(out).not.toContain('The first call shows an inline confirmation card');+ });});describe('isToolVisibleToAgent (ownerAgent gate)', () => {