Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
import type { AkeruToolExecution } from "../provider/AkeruToolRuntime.ts";
import * as Predicate from "effect/Predicate";
import * as NodeCrypto from "node:crypto";
import * as FileSystem from "effect/FileSystem";
import * as Path from "effect/Path";
import { readImageFile } from "./ImageFile.ts";
import {
AkeruUsageReservationId,
type BotId,
type ChatImageAttachment,
CommandId,
decodeImageGenerationRequest,
type ImageArtifact,
type ImageGenerationResult,
type ImageProviderId,
MessageId,
PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,
ProviderDriverKind,
type ThreadId,
type TurnId,
} from "@akeru/contracts";
import * as Context from "effect/Context";
import * as DateTime from "effect/DateTime";
import type * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Fiber from "effect/Fiber";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import { createAttachmentId, resolveAttachmentPath } from "../attachmentStore.ts";
import { ServerConfig } from "../config.ts";
import { OrchestrationEngineService } from "../orchestration/Services/OrchestrationEngine.ts";
import { ProjectionSnapshotQuery } from "../orchestration/Services/ProjectionSnapshotQuery.ts";
import { ProjectionThreadMessageRepositoryLive } from "../persistence/Layers/ProjectionThreadMessages.ts";
import { ProjectionBotRepository } from "../persistence/Services/ProjectionBots.ts";
import { ProjectionThreadMessageRepository } from "../persistence/Services/ProjectionThreadMessages.ts";
import { ServerSettingsService } from "../serverSettings.ts";
import { SubscriptionAuthService } from "../subscription-auth/service.ts";
import { BotUsageLedger } from "../usage/BotUsageLedger.ts";
import {
type ImageAdapterFailure,
type ImageAdapterInputImage,
type ImageProviderAdapter,
chatGptImageAdapter,
grokImageAdapter,
} from "./adapters.ts";
import { sniffImage } from "./imageBytes.ts";
import { type ImageProviderAvailability, routeImageRequest } from "./router.ts";
const EXTENSION_BY_MIME = {
"image/png": ".png",
"image/jpeg": ".jpg",
"image/webp": ".webp",
} as const;
const SUBSCRIPTION_BY_IMAGE_PROVIDER = { chatgpt: "openai-codex", grok: "xai" } as const;
export interface ImageGenerationRuntimeShape {
/**
* Runs one decoded-or-raw tool input for `threadId`. Never fails: every
* problem becomes a `failed` result the calling bot can read. Interrupting
* the effect aborts the provider request.
*/
readonly generate: (
threadId: ThreadId,
input: AkeruToolExecution["input"],
) => Effect.Effect<ImageGenerationResult>;
/** Interrupts every in-flight image request for the chat. */
readonly cancelThread: (threadId: ThreadId) => Effect.Effect<void>;
}
export class ImageGenerationRuntime extends Context.Service<
ImageGenerationRuntime,
ImageGenerationRuntimeShape
>()("akeru-bot/image-generation/ImageGenerationRuntime") {}
/** The subscription store methods the runtime reads for availability and health. */
export type ImageSubscriptionAuth = Pick<
SubscriptionAuthService,
"statuses" | "recordImageGenerationSuccess" | "recordImageRequestFailure"
>;
export interface ImageGenerationRuntimeOptions {
/** Test seams; production builds both from the subscription store. */
readonly adapters?: Readonly<Record<ImageProviderId, ImageProviderAdapter>>;
readonly subscriptionAuth?: ImageSubscriptionAuth;
/** Per-attempt provider timeout; defaults to the router's `IMAGE_REQUEST_TIMEOUT`. */
readonly requestTimeout?: Duration.Input;
}
function failed(
kind: Extract<ImageGenerationResult, { status: "failed" }>["kind"],
message: string,
): ImageGenerationResult {
return { status: "failed", kind, message, attempts: [] };
}
const persistImage = Effect.fn("persistImage")(function* (
fs: FileSystem.FileSystem,
path: Path.Path,
input: {
readonly attachmentsDir: string;
readonly threadId: ThreadId;
readonly bytes: Uint8Array;
readonly mimeType: keyof typeof EXTENSION_BY_MIME;
readonly index: number;
readonly unpostedPaths: Set<string>;
},
): Effect.fn.Return<ChatImageAttachment | null> {
const attachmentId = createAttachmentId(input.threadId);
if (!attachmentId) return null;
const extension = EXTENSION_BY_MIME[input.mimeType];
const finalPath = path.join(input.attachmentsDir, `${attachmentId}${extension}`);
const temporaryPath = `${finalPath}.part`;
input.unpostedPaths.add(temporaryPath);
input.unpostedPaths.add(finalPath);
const saved = yield* Effect.gen(function* () {
yield* fs.makeDirectory(input.attachmentsDir, { recursive: true });
yield* fs.writeFile(temporaryPath, input.bytes, { flag: "wx" });
yield* fs.rename(temporaryPath, finalPath);
return true;
}).pipe(
Effect.catch(() =>
fs.remove(temporaryPath, { force: true }).pipe(Effect.orDie, Effect.as(false)),
),
);
if (!saved) return null;
return {
type: "image",
id: attachmentId,
name: `generated-image-${input.index + 1}${extension}`,
mimeType: input.mimeType,
sizeBytes: input.bytes.byteLength,
};
}, Effect.uninterruptible);
export const makeImageGenerationRuntime = Effect.fn("makeImageGenerationRuntime")(function* (
options: ImageGenerationRuntimeOptions = {},
) {
const config = yield* ServerConfig;
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const engine = yield* OrchestrationEngineService;
const snapshots = yield* ProjectionSnapshotQuery;
const messages = yield* ProjectionThreadMessageRepository;
const bots = yield* ProjectionBotRepository;
const serverSettings = yield* ServerSettingsService;
const usageLedger = yield* BotUsageLedger;
const sharedStore = yield* Effect.cached(
SubscriptionAuthService.forSecretsDir(config.secretsDir),
);
const subscriptionAuth = options.subscriptionAuth ?? (yield* sharedStore);
const adapters = options.adapters ?? {
chatgpt: chatGptImageAdapter({ subscriptionAuth: yield* sharedStore }),
grok: grokImageAdapter({ subscriptionAuth: yield* sharedStore }),
};
const inFlight = new Map<string, Set<Fiber.Fiber<ImageGenerationResult>>>();
// Requests run detached from their caller, so shutdown interrupts them explicitly
// before anything else can write an attachment into a closing server.
yield* Effect.addFinalizer(() =>
Effect.forEach(
[...inFlight.values()].flatMap((fibers) => [...fibers]),
Fiber.interrupt,
{
discard: true,
},
),
);
const availability = (provider: ImageProviderId): ImageProviderAvailability => {
const label = provider === "chatgpt" ? "ChatGPT" : "Grok";
const subscription = subscriptionAuth
.statuses()
.find((status) => status.provider === SUBSCRIPTION_BY_IMAGE_PROVIDER[provider]);
if (!subscription?.connected) {
return {
state: "unavailable",
kind: "unavailable",
message: `${label} is not connected. Connect it in Settings.`,
};
}
if (subscription.health === "revoked") {
return {
state: "unavailable",
kind: "revoked",
message: `${label} access was revoked. Reconnect it in Settings.`,
};
}
return { state: "available" };
};
const recordAttempt = (
provider: ImageProviderId,
outcome: { readonly ok: true } | { readonly ok: false; readonly failure: ImageAdapterFailure },
) =>
Effect.sync(() => {
if (outcome.ok) {
subscriptionAuth.recordImageGenerationSuccess(provider);
return;
}
// Cancellation and bad requests say nothing about the provider's health.
const kind = outcome.failure.kind;
if (kind === "cancelled" || kind === "invalid-request" || kind === "unsupported") return;
subscriptionAuth.recordImageRequestFailure(
provider,
outcome.failure.message,
undefined,
kind === "revoked" ? "revoked" : "request",
);
});
/** Reads the requested input images, which must be image attachments in this chat. */
const resolveInputImages = (threadId: ThreadId, requested: ReadonlyArray<string> | undefined) =>
Effect.gen(function* () {
const threadMessages = yield* messages.listByThreadId({ threadId });
const images = threadMessages.flatMap((message) =>
(message.attachments ?? []).flatMap((attachment) =>
attachment.type === "image" ? [{ attachment, role: message.role }] : [],
),
);
let selected: ReadonlyArray<ChatImageAttachment>;
if (requested) {
const byId = new Map(images.map(({ attachment }) => [attachment.id, attachment]));
const missing = requested.filter((id) => !byId.has(id));
if (missing.length > 0) {
return { ok: false as const, message: "Input images must be images from this chat." };
}
selected = requested.map((id) => byId.get(id)!);
} else {
// Only the latest user message counts, so a text-only follow-up never edits an older image.
const latestUser = threadMessages.findLast((message) => message.role === "user");
selected = (latestUser?.attachments ?? []).filter(
(attachment): attachment is ChatImageAttachment => attachment.type === "image",
);
if (selected.length === 0) {
return {
ok: false as const,
message: "The latest message has no image to edit. Name the images in inputImages.",
};
}
}
const loaded: ImageAdapterInputImage[] = [];
for (const attachment of selected) {
const path = resolveAttachmentPath({ attachmentsDir: config.attachmentsDir, attachment });
const bytes = path
? yield* readImageFile(path).pipe(Effect.provideService(FileSystem.FileSystem, fs))
: null;
if (!bytes) {
return { ok: false as const, message: "An input image is no longer available." };
}
loaded.push({ mimeType: attachment.mimeType, bytes });
}
return { ok: true as const, images: loaded };
});
const postAttachments = (input: {
readonly threadId: ThreadId;
readonly turnId: TurnId | null;
readonly generationId: string;
readonly attachments: ReadonlyArray<ChatImageAttachment>;
readonly unpostedPaths: Set<string>;
}) =>
Effect.gen(function* () {
const createdAt = DateTime.formatIso(yield* DateTime.now);
const messageId = MessageId.make(`image-generation-${input.generationId}`);
yield* Effect.gen(function* () {
yield* engine.dispatch({
type: "thread.message.assistant.delta",
commandId: CommandId.make(`server:image-generation:${input.generationId}`),
threadId: input.threadId,
messageId,
delta: "",
attachments: [...input.attachments],
...(input.turnId ? { turnId: input.turnId } : {}),
createdAt,
});
input.unpostedPaths.clear();
}).pipe(Effect.uninterruptible);
yield* engine.dispatch({
type: "thread.message.assistant.complete",
commandId: CommandId.make(`server:image-generation-complete:${input.generationId}`),
threadId: input.threadId,
messageId,
...(input.turnId ? { turnId: input.turnId } : {}),
createdAt,
});
});
const recordUsage = (input: {
readonly botId: BotId;
readonly threadId: ThreadId;
readonly turnId: TurnId | null;
readonly usageId: string;
readonly provider: ImageProviderId;
readonly model: string | undefined;
readonly usage: { inputTokens: number; outputTokens: number; reasoningTokens: number | null };
}) =>
Effect.gen(function* () {
const createdAt = DateTime.formatIso(yield* DateTime.now);
yield* usageLedger.recordMeasurement({
reservationId: AkeruUsageReservationId.make(`image:${input.usageId}`),
sourceKey: `image:${input.usageId}`,
botId: input.botId,
threadId: input.threadId,
turnId: input.turnId,
category: "tool",
inputTokens: input.usage.inputTokens,
outputTokens: input.usage.outputTokens,
reasoningTokens: input.usage.reasoningTokens,
provider: ProviderDriverKind.make(input.provider),
model: input.model ?? null,
createdAt,
});
});
const run = (threadId: ThreadId, rawInput: AkeruToolExecution["input"]) =>
Effect.gen(function* () {
const decoded = decodeImageGenerationRequest(rawInput);
if (!decoded.ok) return failed("invalid-request", decoded.message);
const request = decoded.request;
const shell = yield* snapshots.getThreadShellById(threadId);
if (Option.isNone(shell)) return failed("invalid-request", "This chat no longer exists.");
const thread = shell.value;
const botId =
thread.latestTurn?.respondingBotId ?? thread.respondingBotId ?? thread.botId ?? null;
const bot = botId ? yield* bots.getById({ botId }) : Option.none();
const turnId = thread.latestTurn?.state === "running" ? thread.latestTurn.turnId : null;
const settings = (yield* serverSettings.getSettings).imageGeneration;
const inputImages =
request.operation === "edit" || request.inputImages
? yield* resolveInputImages(threadId, request.inputImages)
: { ok: true as const, images: [] };
if (!inputImages.ok) return failed("invalid-request", inputImages.message);
const generationId = NodeCrypto.randomUUID();
const routed = yield* routeImageRequest({
request,
inputImages: inputImages.images,
settings,
botOverride: Option.isSome(bot) ? bot.value.imageProvider : null,
availability,
adapters,
...(options.requestTimeout ? { timeout: options.requestTimeout } : {}),
onAttempt: recordAttempt,
onOutput: (provider, output, requestIndex) =>
botId
? recordUsage({
botId,
threadId,
turnId,
usageId: `${generationId}:${requestIndex}`,
provider,
model: output.model,
usage: output.usage ?? {
inputTokens: 0,
outputTokens: 0,
reasoningTokens: null,
},
}).pipe(
Effect.catch(() =>
Effect.logWarning("Could not record image generation usage.", { threadId }),
),
)
: Effect.void,
});
const artifacts: ImageArtifact[] = [];
const attachments: ChatImageAttachment[] = [];
const unpostedPaths = new Set<string>();
yield* Effect.gen(function* () {
for (const part of routed.parts) {
for (const bytes of part.output.images) {
const sniffed = sniffImage(bytes);
if (!sniffed || bytes.byteLength > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) continue;
const attachment = yield* persistImage(fs, path, {
attachmentsDir: config.attachmentsDir,
threadId,
bytes,
mimeType: sniffed.mimeType,
index: attachments.length,
unpostedPaths,
});
if (!attachment) continue;
attachments.push(attachment);
artifacts.push({
attachmentId: attachment.id,
mimeType: sniffed.mimeType,
width: sniffed.width,
height: sniffed.height,
sizeBytes: bytes.byteLength,
provider: part.provider,
...(part.output.model ? { model: part.output.model } : {}),
});
}
}
if (attachments.length > 0) {
yield* postAttachments({ threadId, turnId, generationId, attachments, unpostedPaths });
}
}).pipe(
Effect.ensuring(
Effect.suspend(() =>
Effect.forEach(
unpostedPaths,
(filePath) => fs.remove(filePath, { force: true }).pipe(Effect.orDie),
{ discard: true },
),
),
),
);
if (routed.status !== "completed") {
const { parts: _parts, ...result } = routed;
return result;
}
const requested = request.count ?? 1;
if (attachments.length < requested) {
return {
status: "failed",
kind: "provider-failed",
message:
attachments.length === 0
? "The provider returned no usable image."
: `Only ${attachments.length} of ${requested} images were usable. They are posted in the chat.`,
attempts: routed.attempts,
} satisfies ImageGenerationResult;
}
return {
status: "completed",
provider: routed.provider,
...(routed.parts[0]?.output.model ? { model: routed.parts[0].output.model } : {}),
artifacts,
attempts: routed.attempts,
} satisfies ImageGenerationResult;
}).pipe(
Effect.orElseSucceed(() => failed("provider-failed", "Image generation could not finish.")),
Effect.onInterrupt(() =>
Effect.logInfo("Image generation cancelled.", { threadId: String(threadId) }),
),
);
const generate: ImageGenerationRuntimeShape["generate"] = (threadId, input) =>
Effect.gen(function* () {
const fiber = yield* run(threadId, input).pipe(Effect.forkDetach);
const key = String(threadId);
const fibers = inFlight.get(key) ?? new Set();
fibers.add(fiber);
inFlight.set(key, fibers);
const exit = yield* Fiber.await(fiber).pipe(
Effect.onInterrupt(() => Fiber.interrupt(fiber)),
Effect.ensuring(
Effect.sync(() => {
fibers.delete(fiber);
if (fibers.size === 0) inFlight.delete(key);
}),
),
);
return Predicate.isTagged(exit, "Success")
? exit.value
: failed("cancelled", "Image generation was cancelled.");
});
const cancelThread: ImageGenerationRuntimeShape["cancelThread"] = (threadId) =>
Effect.suspend(() => {
const fibers = [...(inFlight.get(String(threadId)) ?? [])];
return Effect.forEach(fibers, (fiber) => Fiber.interrupt(fiber), { discard: true });
});
return ImageGenerationRuntime.of({ generate, cancelThread });
});
let activeImageGenerationRuntime: ImageGenerationRuntimeShape | undefined;
/**
* Publishes a runtime for tool handlers until the scope closes. MCP handlers
* run without the server's runtime services (route-only test layers build
* `/mcp` alone), so they reach the runtime through this slot.
*/
export const activateImageGenerationRuntime = (runtime: ImageGenerationRuntimeShape) =>
Effect.acquireRelease(
Effect.sync(() => {
activeImageGenerationRuntime = runtime;
}),
() =>
Effect.sync(() => {
if (activeImageGenerationRuntime === runtime) activeImageGenerationRuntime = undefined;
}),
);
export const layerWith = (options: ImageGenerationRuntimeOptions = {}) =>
Layer.effect(
ImageGenerationRuntime,
makeImageGenerationRuntime(options).pipe(
Effect.tap((runtime) => activateImageGenerationRuntime(runtime)),
),
).pipe(Layer.provide(ProjectionThreadMessageRepositoryLive));
export const layer = layerWith();
/** Runs the image tool through the active runtime; the entry point for tool registrations. */
export const runImageGenerationTool = (
threadId: ThreadId,
input: AkeruToolExecution["input"],
): Effect.Effect<ImageGenerationResult> =>
activeImageGenerationRuntime
? activeImageGenerationRuntime.generate(threadId, input)
: Effect.succeed(failed("unavailable", "Image generation is not running on this server."));
/** Cancels in-flight image requests for a chat, for turn interrupts. */
export const cancelActiveImageGenerations = (threadId: ThreadId): Effect.Effect<void> =>
activeImageGenerationRuntime ? activeImageGenerationRuntime.cancelThread(threadId) : Effect.void;