Skip to content

armgit / Orkas

Public Imported from GitHub
0

Orkas is an open-source (MIT), local-first AI team for your desktop: a Commander turns your goal into to-dos and coordinates 16 built-in specialist agents plus Claude Code, Codex, OpenCode, OpenClaw and Hermes, with projects, scheduled automations and connectors. Free app for macOS & Windows; built-in models or your own keys.

Preserve image sizing and video input contracts

wytdb68af147829
diff --git a/src/main/features/image_size.ts b/src/main/features/image_size.ts
new file mode 100644
index 00000000..34a577f4
--- /dev/null
+++ b/src/main/features/image_size.ts
@@ -0,0 +1,81 @@
+/** Provider image size contracts; unsupported requests never resize. */
+export class UnsupportedImageSizeError extends Error {}
+
+export function isSingleImageSeedream(model: string): boolean {
+ return model === 'doubao-seedream-5-0-pro-260628'
+ || model === 'doubao-seedream-5-0-flash-260915';
+}
+
+export function normaliseSeedreamSize(size: string, model: string): string {
+ const s = String(size || '').trim().toLowerCase();
+ if (!s) return '2k';
+ const singleImage = isSingleImageSeedream(model);
+ const invalid = (): string => {
+ if (singleImage) throw new UnsupportedImageSizeError(
+ 'Unsupported image size. Flash/Pro support 1k, 1.5k, 2k, or WIDTHxHEIGHT '
+ + '(921600-4624220 pixels; aspect ratio 1:16-16:1). The request was not resized.',
+ );
+ return '2k'; // Preserve the legacy Lite/older-model compatibility contract.
+ };
+ const keywords = singleImage ? ['1k', '1.5k', '2k'] : ['2k', '3k', '4k'];
+ if (keywords.includes(s)) return s;
+ const m = s.length <= 32 ? s.match(/^(\d+)x(\d+)$/) : null;
+ if (!m) return invalid();
+ const w = Number(m[1]);
+ const h = Number(m[2]);
+ const pixels = w * h;
+ const aspect = w / h;
+ const minPixels = singleImage ? 921_600 : 3_686_400;
+ const maxPixels = singleImage ? 4_624_220 : 16_777_216;
+ if (!w || !h || pixels < minPixels || pixels > maxPixels || aspect < 1 / 16 || aspect > 16) return invalid();
+ return `${w}x${h}`;
+}
+
+export function normaliseManagedImageSize(size: string): string {
+ return normaliseSeedreamSize(size, 'doubao-seedream-5-0-flash-260915');
+}
+
+type GeminiImageConfig = { aspectRatio: string; imageSize?: '512' | '1K' | '2K' | '4K' };
+
+// Exact pixel pairs from the GenerateContent tables (not mathematical aspect
+// approximations). The documented 2K/4K pairs are exactly 2x/4x these 1K pairs.
+// https://ai.google.dev/gemini-api/docs/generate-content/image-generation
+const GEMINI_1K_SIZES: ReadonlyArray<readonly [string, number, number]> = [
+ ['1:1', 1024, 1024], ['1:4', 512, 2048], ['1:8', 384, 3072],
+ ['2:3', 848, 1264], ['3:2', 1264, 848], ['3:4', 896, 1200],
+ ['4:1', 2048, 512], ['4:3', 1200, 896], ['4:5', 928, 1152],
+ ['5:4', 1152, 928], ['8:1', 3072, 384], ['9:16', 768, 1376],
+ ['16:9', 1376, 768], ['21:9', 1584, 672],
+];
+const GEMINI_25_SIZES: ReadonlyArray<readonly [string, number, number]> = [
+ ['1:1', 1024, 1024], ['2:3', 832, 1248], ['3:2', 1248, 832],
+ ['3:4', 864, 1184], ['4:3', 1184, 864], ['4:5', 896, 1152],
+ ['5:4', 1152, 896], ['9:16', 768, 1344], ['16:9', 1344, 768], ['21:9', 1536, 672],
+];
+
+export function normaliseGeminiImageSize(size: string, model: string): GeminiImageConfig | undefined {
+ const value = String(size || '').trim().toLowerCase();
+ if (!value || value === 'auto') return undefined;
+ const legacy = model === 'gemini-2.5-flash-image' || model === 'gemini-2.5-flash-image-preview';
+ const flash = model === 'gemini-3.1-flash-image' || model === 'gemini-3.1-flash-image-preview';
+ const nano = model === 'gemini-nano-banana-2.1';
+ const pro = model === 'gemini-3-pro-image' || model === 'gemini-3-pro-image-preview'
+ || model === 'gemini-3.1-pro-image';
+ if (legacy || flash || nano || pro) {
+ for (const [aspectRatio, width, height] of legacy ? GEMINI_25_SIZES : GEMINI_1K_SIZES) {
+ if (pro && ['1:4', '1:8', '4:1', '8:1'].includes(aspectRatio)) continue;
+ if (legacy) {
+ if (value === `${width}x${height}`) return { aspectRatio };
+ continue;
+ }
+ for (const [scale, imageSize] of [[0.5, '512'], [1, '1K'], [2, '2K'], [4, '4K']] as const) {
+ if (scale === 0.5 && !flash) continue;
+ if (value === `${width * scale}x${height * scale}`) return { aspectRatio, imageSize };
+ }
+ }
+ }
+ throw new UnsupportedImageSizeError(
+ 'Unsupported image size for the selected Gemini model. Use auto or an exact WIDTHxHEIGHT '
+ + 'from its supported size table. The request was not resized.',
+ );
+}
diff --git a/src/main/features/video_studio_contract.ts b/src/main/features/video_studio_contract.ts
index efb11fdb..3883e212 100644
--- a/src/main/features/video_studio_contract.ts
+++ b/src/main/features/video_studio_contract.ts
@@ -84,7 +84,7 @@ export const CompositionManifestSchema = z.object({
source_alignment: z.object({
merge_reason: z.string().trim().min(1).optional(),
}).strict().optional(),
- art_direction: z.record(z.unknown()).optional(),
+ art_direction: z.record(z.string(), z.unknown()).optional(),
}).strict().superRefine((manifest, ctx) => {
const hasNarration = manifest.scenes.some((scene) => !!scene.narration_text?.trim());
if (manifest.schema_version === 2
diff --git a/src/main/features/video_vision.ts b/src/main/features/video_vision.ts
index dc45dcb6..3658b5dd 100644
--- a/src/main/features/video_vision.ts
+++ b/src/main/features/video_vision.ts
@@ -30,7 +30,7 @@
import * as fs from 'node:fs/promises';
import { chatWithModel } from '../model/client';
-import { toCompressedGrayJpeg } from '../util/image-transform';
+import { toCompressedJpeg } from '../util/image-transform';
import { createLogger } from '../logger';
import { logErrorSummary, logPathRef } from '../util/log-redact';
@@ -139,7 +139,7 @@ function buildVisionMessage(question: string, frameCount: number): string {
async function loadFrame(p: string): Promise<LoadedImage | null> {
try {
const raw = await fs.readFile(p);
- const compressed = await toCompressedGrayJpeg(raw, { maxDim: FRAME_MAX_DIM, quality: FRAME_JPEG_QUALITY, grayscale: false });
+ const compressed = await toCompressedJpeg(raw, { maxDim: FRAME_MAX_DIM, quality: FRAME_JPEG_QUALITY });
return { data: compressed.buf.toString('base64'), mediaType: 'image/jpeg' };
} catch (err) {
log.warn('frame load failed', { frame: logPathRef(p), error: logErrorSummary(err) });
diff --git a/src/main/util/image-transform.ts b/src/main/util/image-transform.ts
index 76fc9941..431d8ef1 100644
--- a/src/main/util/image-transform.ts
+++ b/src/main/util/image-transform.ts
@@ -1,9 +1,8 @@
/**
* Shared image preprocessing utilities for main/.
*
- * Knowledge-base indexing uses the grayscale JPEG path because vision input
- * only needs text/shapes/layout. User feedback keeps color and only compresses
- * upload payloads when doing so is useful.
+ * Preserve color in model inputs while bounding image dimensions and JPEG
+ * quality. User feedback only compresses upload payloads when useful.
*/
export interface ImageTransformOpts {
@@ -11,8 +10,6 @@ export interface ImageTransformOpts {
maxDim?: number;
/** JPEG quality 1-100. Default 70. */
quality?: number;
- /** Apply greyscale. Default true. */
- grayscale?: boolean;
}
export interface ImageTransformResult {
@@ -27,7 +24,6 @@ export interface ImageTransformResult {
export const LIBRARY_IMAGE_TRANSFORM_OPTIONS: Readonly<Required<ImageTransformOpts>> = Object.freeze({
maxDim: 1024,
quality: 70,
- grayscale: true,
});
export interface FeedbackImageTransformOpts {
@@ -82,16 +78,15 @@ function flattenAlphaOnWhite(Jimp: any, img: any): { image: any; flattened: bool
return { image: background, flattened: true };
}
-export async function toCompressedGrayJpeg(
+export async function toCompressedJpeg(
buf: Buffer,
opts: ImageTransformOpts = {},
): Promise<ImageTransformResult> {
if (!buf || !(buf instanceof Buffer) || buf.length === 0) {
- throw new Error('toCompressedGrayJpeg: empty or invalid buffer');
+ throw new Error('toCompressedJpeg: empty or invalid buffer');
}
const maxDim = Math.max(64, opts.maxDim ?? 1024);
const quality = Math.min(100, Math.max(1, opts.quality ?? 70));
- const grayscale = opts.grayscale !== false;
const Jimp: any = await loadJimp();
let img: any = await Jimp.read(buf);
@@ -106,7 +101,6 @@ export async function toCompressedGrayJpeg(
if (img.bitmap.width > maxDim || img.bitmap.height > maxDim) {
img.scaleToFit({ w: maxDim, h: maxDim });
}
- if (grayscale) img.greyscale();
const out: Buffer = await img.getBuffer('image/jpeg', { quality });
return {
diff --git a/test/main/features/video_vision.test.ts b/test/main/features/video_vision.test.ts
index 5a216fba..71c3ce45 100644
--- a/test/main/features/video_vision.test.ts
+++ b/test/main/features/video_vision.test.ts
@@ -1,4 +1,5 @@
import { getEventListeners } from 'node:events';
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { describe, expect, it, vi } from 'vitest';
@@ -149,6 +150,29 @@ describe('analyzeFrames — orchestration', () => {
if (failure === 'frame') expect(diagnostic[1].frame).toMatchObject({ path_hash: expect.stringMatching(/^[a-f0-9]{12}$/) });
});
+ it('preserves color through the real frame loader into model input', async () => {
+ const dir = await mkdtemp(join(tmpdir(), 'orkas-frame-color-'));
+ try {
+ const { Jimp } = await import('jimp' as any);
+ const frame = new Jimp({ width: 80, height: 60, color: 0xDC1E1EFF });
+ const path = join(dir, 'frame.png');
+ await writeFile(path, await frame.getBuffer('image/png'));
+ const chat = vi.fn(async (_opts: { images?: LoadedImage[] }) => okRes('A red frame.'));
+ const result = await analyzeFrames({
+ userId: 'u1', framePaths: [path], question: 'describe', chat: chat as never,
+ });
+ expect(result).toMatchObject({ ok: true, framesUsed: 1 });
+ const options = chat.mock.calls[0][0];
+ const decoded = await Jimp.read(Buffer.from(options.images![0].data, 'base64'));
+ const px = decoded.getPixelColor(40, 30);
+ expect((px >>> 24) & 0xff).toBeGreaterThan(200);
+ expect((px >>> 16) & 0xff).toBeLessThan(45);
+ expect((px >>> 8) & 0xff).toBeLessThan(45);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+ });
+
it('degrades to no-frames when no paths are supplied (no model call)', async () => {
const chat = vi.fn();
const r = await analyzeFrames({ userId: 'u1', framePaths: [], question: 'what?', chat: chat as never });
diff --git a/test/main/util/image-transform.test.ts b/test/main/util/image-transform.test.ts
index 3698ff45..cd53d57b 100644
--- a/test/main/util/image-transform.test.ts
+++ b/test/main/util/image-transform.test.ts
@@ -2,7 +2,7 @@ import { describe, it, expect, beforeAll } from 'vitest';
import {
flattenTransparentImageToJpeg,
prepareFeedbackUploadImage,
- toCompressedGrayJpeg,
+ toCompressedJpeg,
} from '../../../src/main/util/image-transform';
let sourcePng: Buffer;
@@ -11,7 +11,7 @@ beforeAll(async () => {
// Build a 2000x1500 colorful PNG via jimp (test-only dep on the same lib).
const { Jimp } = await import('jimp' as any);
const img: any = new Jimp({ width: 2000, height: 1500, color: 0x336699FF });
- // Stamp a few colored bands so greyscale conversion has something to verify.
+ // Include red and blue regions to detect lost color after compression.
img.scan(0, 0, 2000, 100, function (this: any, _x: number, _y: number, idx: number) {
this.bitmap.data[idx + 0] = 220;
this.bitmap.data[idx + 1] = 30;
@@ -20,9 +20,9 @@ beforeAll(async () => {
sourcePng = await img.getBuffer('image/png');
});
-describe('image-transform › toCompressedGrayJpeg', () => {
+describe('image-transform › toCompressedJpeg', () => {
it('downscales long edge to maxDim and emits JPEG', async () => {
- const r = await toCompressedGrayJpeg(sourcePng, { maxDim: 800, quality: 60 });
+ const r = await toCompressedJpeg(sourcePng, { maxDim: 800, quality: 60 });
expect(r.mimeType).toBe('image/jpeg');
expect(Math.max(r.width, r.height)).toBeLessThanOrEqual(800);
// JPEG magic
@@ -31,24 +31,25 @@ describe('image-transform › toCompressedGrayJpeg', () => {
expect(r.buf.length).toBeGreaterThan(100);
});
- it('produces grayscale pixels (R≈G≈B for sampled pixel)', async () => {
- const r = await toCompressedGrayJpeg(sourcePng, { maxDim: 200 });
+ it('preserves red and blue pixels with the default transform', async () => {
+ const r = await toCompressedJpeg(sourcePng);
const { Jimp } = await import('jimp' as any);
const out: any = await Jimp.read(r.buf);
- // Sample a band that started red — after greyscale, R/G/B should be near-equal.
- const px = out.getPixelColor(50, 50); // 0xRRGGBBAA
- const r8 = (px >>> 24) & 0xff;
- const g8 = (px >>> 16) & 0xff;
- const b8 = (px >>> 8) & 0xff;
- expect(Math.abs(r8 - g8)).toBeLessThanOrEqual(8);
- expect(Math.abs(g8 - b8)).toBeLessThanOrEqual(8);
+ for (const [x, y, expected] of [
+ [50, 20, [220, 30, 30]],
+ [50, 100, [51, 102, 153]],
+ ] as const) {
+ const px = out.getPixelColor(x, y);
+ const actual = [(px >>> 24) & 0xff, (px >>> 16) & 0xff, (px >>> 8) & 0xff];
+ actual.forEach((channel, i) => expect(Math.abs(channel - expected[i])).toBeLessThan(15));
+ }
});
it('does not upscale when source is smaller than maxDim', async () => {
const { Jimp } = await import('jimp' as any);
const small: any = new Jimp({ width: 100, height: 80, color: 0xFFFFFFFF });
const smallPng: Buffer = await small.getBuffer('image/png');
- const r = await toCompressedGrayJpeg(smallPng, { maxDim: 1024 });
+ const r = await toCompressedJpeg(smallPng, { maxDim: 1024 });
expect(r.width).toBe(100);
expect(r.height).toBe(80);
});
@@ -64,7 +65,7 @@ describe('image-transform › toCompressedGrayJpeg', () => {
});
const input: Buffer = await transparent.getBuffer('image/png');
- const result = await toCompressedGrayJpeg(input, { maxDim: 100, quality: 90 });
+ const result = await toCompressedJpeg(input, { maxDim: 100, quality: 90 });
const output: any = await Jimp.read(result.buf);
const background = output.getPixelColor(5, 5);
const foreground = output.getPixelColor(40, 30);
@@ -84,7 +85,7 @@ describe('image-transform › toCompressedGrayJpeg', () => {
});
it('rejects empty buffer', async () => {
- await expect(toCompressedGrayJpeg(Buffer.alloc(0))).rejects.toThrow(/empty|invalid/i);
+ await expect(toCompressedJpeg(Buffer.alloc(0))).rejects.toThrow(/empty|invalid/i);
});
});