Orkas is an open-source (MIT), local-first AI team for your desktop: a Commander turns your goal into to-dos and coordinates 16 built-in specialist agents plus Claude Code, Codex, OpenCode, OpenClaw and Hermes, with projects, scheduled automations and connectors. Free app for macOS & Windows; built-in models or your own keys.
JavaScript agent-collaborationagent-harnessagent-orchestrationagent-teamagent-workflowai-assistantai-automationai-employeeai-productivityai-teamai-workflowautonomous-agentsclaude-code-guiclaude-coworkdesktop-agentlocal-firstmulti-agentmulti-agent-systemself-evolving-agentstask-automation
Preserve image sizing and video input contracts
wytdb68af147829
diff --git a/src/main/features/image_size.ts b/src/main/features/image_size.tsnew file mode 100644index 00000000..34a577f4--- /dev/null+++ b/src/main/features/image_size.ts@@ -0,0 +1,81 @@+/** Provider image size contracts; unsupported requests never resize. */+export class UnsupportedImageSizeError extends Error {}++export function isSingleImageSeedream(model: string): boolean {+ return model === 'doubao-seedream-5-0-pro-260628'+ || model === 'doubao-seedream-5-0-flash-260915';+}++export function normaliseSeedreamSize(size: string, model: string): string {+ const s = String(size || '').trim().toLowerCase();+ if (!s) return '2k';+ const singleImage = isSingleImageSeedream(model);+ const invalid = (): string => {+ if (singleImage) throw new UnsupportedImageSizeError(+ 'Unsupported image size. Flash/Pro support 1k, 1.5k, 2k, or WIDTHxHEIGHT '+ + '(921600-4624220 pixels; aspect ratio 1:16-16:1). The request was not resized.',+ );+ return '2k'; // Preserve the legacy Lite/older-model compatibility contract.+ };+ const keywords = singleImage ? ['1k', '1.5k', '2k'] : ['2k', '3k', '4k'];+ if (keywords.includes(s)) return s;+ const m = s.length <= 32 ? s.match(/^(\d+)x(\d+)$/) : null;+ if (!m) return invalid();+ const w = Number(m[1]);+ const h = Number(m[2]);+ const pixels = w * h;+ const aspect = w / h;+ const minPixels = singleImage ? 921_600 : 3_686_400;+ const maxPixels = singleImage ? 4_624_220 : 16_777_216;+ if (!w || !h || pixels < minPixels || pixels > maxPixels || aspect < 1 / 16 || aspect > 16) return invalid();+ return `${w}x${h}`;+}++export function normaliseManagedImageSize(size: string): string {+ return normaliseSeedreamSize(size, 'doubao-seedream-5-0-flash-260915');+}++type GeminiImageConfig = { aspectRatio: string; imageSize?: '512' | '1K' | '2K' | '4K' };++// Exact pixel pairs from the GenerateContent tables (not mathematical aspect+// approximations). The documented 2K/4K pairs are exactly 2x/4x these 1K pairs.+// https://ai.google.dev/gemini-api/docs/generate-content/image-generation+const GEMINI_1K_SIZES: ReadonlyArray<readonly [string, number, number]> = [+ ['1:1', 1024, 1024], ['1:4', 512, 2048], ['1:8', 384, 3072],+ ['2:3', 848, 1264], ['3:2', 1264, 848], ['3:4', 896, 1200],+ ['4:1', 2048, 512], ['4:3', 1200, 896], ['4:5', 928, 1152],+ ['5:4', 1152, 928], ['8:1', 3072, 384], ['9:16', 768, 1376],+ ['16:9', 1376, 768], ['21:9', 1584, 672],+];+const GEMINI_25_SIZES: ReadonlyArray<readonly [string, number, number]> = [+ ['1:1', 1024, 1024], ['2:3', 832, 1248], ['3:2', 1248, 832],+ ['3:4', 864, 1184], ['4:3', 1184, 864], ['4:5', 896, 1152],+ ['5:4', 1152, 896], ['9:16', 768, 1344], ['16:9', 1344, 768], ['21:9', 1536, 672],+];++export function normaliseGeminiImageSize(size: string, model: string): GeminiImageConfig | undefined {+ const value = String(size || '').trim().toLowerCase();+ if (!value || value === 'auto') return undefined;+ const legacy = model === 'gemini-2.5-flash-image' || model === 'gemini-2.5-flash-image-preview';+ const flash = model === 'gemini-3.1-flash-image' || model === 'gemini-3.1-flash-image-preview';+ const nano = model === 'gemini-nano-banana-2.1';+ const pro = model === 'gemini-3-pro-image' || model === 'gemini-3-pro-image-preview'+ || model === 'gemini-3.1-pro-image';+ if (legacy || flash || nano || pro) {+ for (const [aspectRatio, width, height] of legacy ? GEMINI_25_SIZES : GEMINI_1K_SIZES) {+ if (pro && ['1:4', '1:8', '4:1', '8:1'].includes(aspectRatio)) continue;+ if (legacy) {+ if (value === `${width}x${height}`) return { aspectRatio };+ continue;+ }+ for (const [scale, imageSize] of [[0.5, '512'], [1, '1K'], [2, '2K'], [4, '4K']] as const) {+ if (scale === 0.5 && !flash) continue;+ if (value === `${width * scale}x${height * scale}`) return { aspectRatio, imageSize };+ }+ }+ }+ throw new UnsupportedImageSizeError(+ 'Unsupported image size for the selected Gemini model. Use auto or an exact WIDTHxHEIGHT '+ + 'from its supported size table. The request was not resized.',+ );+}diff --git a/src/main/features/video_studio_contract.ts b/src/main/features/video_studio_contract.tsindex efb11fdb..3883e212 100644--- a/src/main/features/video_studio_contract.ts+++ b/src/main/features/video_studio_contract.ts@@ -84,7 +84,7 @@ export const CompositionManifestSchema = z.object({source_alignment: z.object({merge_reason: z.string().trim().min(1).optional(),}).strict().optional(),- art_direction: z.record(z.unknown()).optional(),+ art_direction: z.record(z.string(), z.unknown()).optional(),}).strict().superRefine((manifest, ctx) => {const hasNarration = manifest.scenes.some((scene) => !!scene.narration_text?.trim());if (manifest.schema_version === 2diff --git a/src/main/features/video_vision.ts b/src/main/features/video_vision.tsindex dc45dcb6..3658b5dd 100644--- a/src/main/features/video_vision.ts+++ b/src/main/features/video_vision.ts@@ -30,7 +30,7 @@import * as fs from 'node:fs/promises';import { chatWithModel } from '../model/client';-import { toCompressedGrayJpeg } from '../util/image-transform';+import { toCompressedJpeg } from '../util/image-transform';import { createLogger } from '../logger';import { logErrorSummary, logPathRef } from '../util/log-redact';@@ -139,7 +139,7 @@ function buildVisionMessage(question: string, frameCount: number): string {async function loadFrame(p: string): Promise<LoadedImage | null> {try {const raw = await fs.readFile(p);- const compressed = await toCompressedGrayJpeg(raw, { maxDim: FRAME_MAX_DIM, quality: FRAME_JPEG_QUALITY, grayscale: false });+ const compressed = await toCompressedJpeg(raw, { maxDim: FRAME_MAX_DIM, quality: FRAME_JPEG_QUALITY });return { data: compressed.buf.toString('base64'), mediaType: 'image/jpeg' };} catch (err) {log.warn('frame load failed', { frame: logPathRef(p), error: logErrorSummary(err) });diff --git a/src/main/util/image-transform.ts b/src/main/util/image-transform.tsindex 76fc9941..431d8ef1 100644--- a/src/main/util/image-transform.ts+++ b/src/main/util/image-transform.ts@@ -1,9 +1,8 @@/*** Shared image preprocessing utilities for main/.*- * Knowledge-base indexing uses the grayscale JPEG path because vision input- * only needs text/shapes/layout. User feedback keeps color and only compresses- * upload payloads when doing so is useful.+ * Preserve color in model inputs while bounding image dimensions and JPEG+ * quality. User feedback only compresses upload payloads when useful.*/export interface ImageTransformOpts {@@ -11,8 +10,6 @@ export interface ImageTransformOpts {maxDim?: number;/** JPEG quality 1-100. Default 70. */quality?: number;- /** Apply greyscale. Default true. */- grayscale?: boolean;}export interface ImageTransformResult {@@ -27,7 +24,6 @@ export interface ImageTransformResult {export const LIBRARY_IMAGE_TRANSFORM_OPTIONS: Readonly<Required<ImageTransformOpts>> = Object.freeze({maxDim: 1024,quality: 70,- grayscale: true,});export interface FeedbackImageTransformOpts {@@ -82,16 +78,15 @@ function flattenAlphaOnWhite(Jimp: any, img: any): { image: any; flattened: boolreturn { image: background, flattened: true };}-export async function toCompressedGrayJpeg(+export async function toCompressedJpeg(buf: Buffer,opts: ImageTransformOpts = {},): Promise<ImageTransformResult> {if (!buf || !(buf instanceof Buffer) || buf.length === 0) {- throw new Error('toCompressedGrayJpeg: empty or invalid buffer');+ throw new Error('toCompressedJpeg: empty or invalid buffer');}const maxDim = Math.max(64, opts.maxDim ?? 1024);const quality = Math.min(100, Math.max(1, opts.quality ?? 70));- const grayscale = opts.grayscale !== false;const Jimp: any = await loadJimp();let img: any = await Jimp.read(buf);@@ -106,7 +101,6 @@ export async function toCompressedGrayJpeg(if (img.bitmap.width > maxDim || img.bitmap.height > maxDim) {img.scaleToFit({ w: maxDim, h: maxDim });}- if (grayscale) img.greyscale();const out: Buffer = await img.getBuffer('image/jpeg', { quality });return {diff --git a/test/main/features/video_vision.test.ts b/test/main/features/video_vision.test.tsindex 5a216fba..71c3ce45 100644--- a/test/main/features/video_vision.test.ts+++ b/test/main/features/video_vision.test.ts@@ -1,4 +1,5 @@import { getEventListeners } from 'node:events';+import { mkdtemp, rm, writeFile } from 'node:fs/promises';import { tmpdir } from 'node:os';import { join } from 'node:path';import { describe, expect, it, vi } from 'vitest';@@ -149,6 +150,29 @@ describe('analyzeFrames — orchestration', () => {if (failure === 'frame') expect(diagnostic[1].frame).toMatchObject({ path_hash: expect.stringMatching(/^[a-f0-9]{12}$/) });});+ it('preserves color through the real frame loader into model input', async () => {+ const dir = await mkdtemp(join(tmpdir(), 'orkas-frame-color-'));+ try {+ const { Jimp } = await import('jimp' as any);+ const frame = new Jimp({ width: 80, height: 60, color: 0xDC1E1EFF });+ const path = join(dir, 'frame.png');+ await writeFile(path, await frame.getBuffer('image/png'));+ const chat = vi.fn(async (_opts: { images?: LoadedImage[] }) => okRes('A red frame.'));+ const result = await analyzeFrames({+ userId: 'u1', framePaths: [path], question: 'describe', chat: chat as never,+ });+ expect(result).toMatchObject({ ok: true, framesUsed: 1 });+ const options = chat.mock.calls[0][0];+ const decoded = await Jimp.read(Buffer.from(options.images![0].data, 'base64'));+ const px = decoded.getPixelColor(40, 30);+ expect((px >>> 24) & 0xff).toBeGreaterThan(200);+ expect((px >>> 16) & 0xff).toBeLessThan(45);+ expect((px >>> 8) & 0xff).toBeLessThan(45);+ } finally {+ await rm(dir, { recursive: true, force: true });+ }+ });+it('degrades to no-frames when no paths are supplied (no model call)', async () => {const chat = vi.fn();const r = await analyzeFrames({ userId: 'u1', framePaths: [], question: 'what?', chat: chat as never });diff --git a/test/main/util/image-transform.test.ts b/test/main/util/image-transform.test.tsindex 3698ff45..cd53d57b 100644--- a/test/main/util/image-transform.test.ts+++ b/test/main/util/image-transform.test.ts@@ -2,7 +2,7 @@ import { describe, it, expect, beforeAll } from 'vitest';import {flattenTransparentImageToJpeg,prepareFeedbackUploadImage,- toCompressedGrayJpeg,+ toCompressedJpeg,} from '../../../src/main/util/image-transform';let sourcePng: Buffer;@@ -11,7 +11,7 @@ beforeAll(async () => {// Build a 2000x1500 colorful PNG via jimp (test-only dep on the same lib).const { Jimp } = await import('jimp' as any);const img: any = new Jimp({ width: 2000, height: 1500, color: 0x336699FF });- // Stamp a few colored bands so greyscale conversion has something to verify.+ // Include red and blue regions to detect lost color after compression.img.scan(0, 0, 2000, 100, function (this: any, _x: number, _y: number, idx: number) {this.bitmap.data[idx + 0] = 220;this.bitmap.data[idx + 1] = 30;@@ -20,9 +20,9 @@ beforeAll(async () => {sourcePng = await img.getBuffer('image/png');});-describe('image-transform › toCompressedGrayJpeg', () => {+describe('image-transform › toCompressedJpeg', () => {it('downscales long edge to maxDim and emits JPEG', async () => {- const r = await toCompressedGrayJpeg(sourcePng, { maxDim: 800, quality: 60 });+ const r = await toCompressedJpeg(sourcePng, { maxDim: 800, quality: 60 });expect(r.mimeType).toBe('image/jpeg');expect(Math.max(r.width, r.height)).toBeLessThanOrEqual(800);// JPEG magic@@ -31,24 +31,25 @@ describe('image-transform › toCompressedGrayJpeg', () => {expect(r.buf.length).toBeGreaterThan(100);});- it('produces grayscale pixels (R≈G≈B for sampled pixel)', async () => {- const r = await toCompressedGrayJpeg(sourcePng, { maxDim: 200 });+ it('preserves red and blue pixels with the default transform', async () => {+ const r = await toCompressedJpeg(sourcePng);const { Jimp } = await import('jimp' as any);const out: any = await Jimp.read(r.buf);- // Sample a band that started red — after greyscale, R/G/B should be near-equal.- const px = out.getPixelColor(50, 50); // 0xRRGGBBAA- const r8 = (px >>> 24) & 0xff;- const g8 = (px >>> 16) & 0xff;- const b8 = (px >>> 8) & 0xff;- expect(Math.abs(r8 - g8)).toBeLessThanOrEqual(8);- expect(Math.abs(g8 - b8)).toBeLessThanOrEqual(8);+ for (const [x, y, expected] of [+ [50, 20, [220, 30, 30]],+ [50, 100, [51, 102, 153]],+ ] as const) {+ const px = out.getPixelColor(x, y);+ const actual = [(px >>> 24) & 0xff, (px >>> 16) & 0xff, (px >>> 8) & 0xff];+ actual.forEach((channel, i) => expect(Math.abs(channel - expected[i])).toBeLessThan(15));+ }});it('does not upscale when source is smaller than maxDim', async () => {const { Jimp } = await import('jimp' as any);const small: any = new Jimp({ width: 100, height: 80, color: 0xFFFFFFFF });const smallPng: Buffer = await small.getBuffer('image/png');- const r = await toCompressedGrayJpeg(smallPng, { maxDim: 1024 });+ const r = await toCompressedJpeg(smallPng, { maxDim: 1024 });expect(r.width).toBe(100);expect(r.height).toBe(80);});@@ -64,7 +65,7 @@ describe('image-transform › toCompressedGrayJpeg', () => {});const input: Buffer = await transparent.getBuffer('image/png');- const result = await toCompressedGrayJpeg(input, { maxDim: 100, quality: 90 });+ const result = await toCompressedJpeg(input, { maxDim: 100, quality: 90 });const output: any = await Jimp.read(result.buf);const background = output.getPixelColor(5, 5);const foreground = output.getPixelColor(40, 30);@@ -84,7 +85,7 @@ describe('image-transform › toCompressedGrayJpeg', () => {});it('rejects empty buffer', async () => {- await expect(toCompressedGrayJpeg(Buffer.alloc(0))).rejects.toThrow(/empty|invalid/i);+ await expect(toCompressedJpeg(Buffer.alloc(0))).rejects.toThrow(/empty|invalid/i);});});