Open source CapCut alternative.
TypeScript
feat(site): generate a post's details and pictures from the article
David Le6c8235584b46
diff --git a/site/src/app/api/su/blog/posts/[id]/generate/route.ts b/site/src/app/api/su/blog/posts/[id]/generate/route.tsnew file mode 100644index 00000000..8314d32d--- /dev/null+++ b/site/src/app/api/su/blog/posts/[id]/generate/route.ts@@ -0,0 +1,101 @@+import { NextResponse } from "next/server";+import { z } from "zod";++import { recordFailedInferenceUsage, recordInferenceUsage } from "@/lib/credits/inference";+import { adminPost } from "@/lib/blog/admin";+import {+ generateBlogDetails,+ generateBlogPicture,+ IMAGE_MODEL,+ TEXT_MODEL,+ type BlogDetails,+} from "@/lib/blog/generate";+import { storeBlogImage } from "@/lib/blog/images";+import { revalidateBlogLater } from "@/lib/blog/revalidate";+import { invalidResponse } from "@/lib/config/experimentList";+import { InferenceProviderError, type JsonValue } from "@/lib/inference/providers";+import { notFoundResponse, shouldBypassDonkeyInferenceCredits, withSuperUser } from "@/lib/donkey-api-auth";+import { prisma } from "@/lib/prisma";++type Params = { params: Promise<{ id: string }> };+const idSchema = z.string().trim().min(1);++const ROUTE = "/api/su/blog/posts/[id]/generate";+const PROVIDER = "gemini";++// The editor sends the article as it stands in the form, saved or not.+const bodySchema = z+ .object({+ title: z.string().trim().max(200),+ body: z.string().trim().min(1, "Write the article first.").max(2_000_000),+ })+ .strict();++// Fills the details from the article: the copy comes back for the form, the+// picture is stored as the header and the thumbnail. Every model call is+// recorded against the editor's account as included usage.+export const POST = withSuperUser(async (request, { params }: Params) => {+ const id = idSchema.safeParse((await params).id);+ if (!id.success) return notFoundResponse();+ const parsed = bodySchema.safeParse(await request.json());+ if (!parsed.success) return invalidResponse(parsed.error.issues);+ const existing = await prisma.blogPost.findUnique({ select: { id: true }, where: { id: id.data } });+ if (!existing) return notFoundResponse();+ // A dev-bypass caller has no account row to record against.+ const recorded = !shouldBypassDonkeyInferenceCredits(request.donkey);+ const usageOf = (requestKind: string, model: string) => ({+ billingMode: "included" as const,+ clientId: null,+ model,+ provider: PROVIDER,+ requestKind,+ route: ROUTE,+ userId: request.donkey.userId,+ });+ const succeeded = async (requestKind: string, model: string, usage: JsonValue) => {+ if (recorded) await recordInferenceUsage({ ...usageOf(requestKind, model), status: "succeeded", usage });+ };+ const failed = async (requestKind: string, model: string) => {+ if (recorded) await recordFailedInferenceUsage({ ...usageOf(requestKind, model), errorCode: "provider_error" });+ };++ let details: BlogDetails;+ try {+ const generated = await generateBlogDetails(parsed.data.title, parsed.data.body);+ details = generated.value;+ await succeeded("blog_details", TEXT_MODEL, generated.usage);+ } catch (error) {+ await failed("blog_details", TEXT_MODEL);+ return failure(error);+ }++ let picture: Buffer;+ try {+ const generated = await generateBlogPicture(details.imagePrompt);+ picture = generated.value;+ await succeeded("blog_picture", IMAGE_MODEL, generated.usage);+ } catch (error) {+ await failed("blog_picture", IMAGE_MODEL);+ return failure(error);+ }++ const [header, thumbnail] = await Promise.all([+ storeBlogImage(existing.id, picture, "header"),+ storeBlogImage(existing.id, picture, "thumbnail"),+ ]);+ const row = await prisma.blogPost.update({+ where: { id: existing.id },+ data: { headerKey: header.key, thumbnailKey: thumbnail.key },+ });+ revalidateBlogLater([row.slug]);++ const { seoTitle, excerpt, summary, tags, keywords, headerAlt } = details;+ return NextResponse.json({ post: adminPost(row), details: { seoTitle, excerpt, summary, tags, keywords, headerAlt } });+});++function failure(error: unknown) {+ if (error instanceof InferenceProviderError) {+ return NextResponse.json({ error: error.message }, { status: error.statusCode ?? 502 });+ }+ throw error;+}diff --git a/site/src/app/api/su/blog/posts/[id]/images/route.ts b/site/src/app/api/su/blog/posts/[id]/images/route.tsindex 1169f9c8..3eedd9bf 100644--- a/site/src/app/api/su/blog/posts/[id]/images/route.ts+++ b/site/src/app/api/su/blog/posts/[id]/images/route.ts@@ -1,10 +1,10 @@import { NextResponse } from "next/server";import { z } from "zod";-import { del, getObject, putObject } from "@/cut/server/cloud/r2";+import { del, getObject } from "@/cut/server/cloud/r2";import { adminPost } from "@/lib/blog/admin";-import { encodeBlogImage } from "@/lib/blog/images";-import { blogImageKey, blogImageUrl, blogPrefix } from "@/lib/blog/keys";+import { storeBlogImage } from "@/lib/blog/images";+import { blogPrefix } from "@/lib/blog/keys";import { revalidateBlogLater } from "@/lib/blog/revalidate";import { invalidResponse } from "@/lib/config/experimentList";import { notFoundResponse, withSuperUser } from "@/lib/donkey-api-auth";@@ -39,17 +39,14 @@ export const POST = withSuperUser(async (request, { params }: Params) => {const upload = await getObject(uploadKey);if (!upload) return invalidResponse([{ path: ["uploadKey"], message: "The upload was not found." }]);- const encoded = await encodeBlogImage(upload.bytes, kind);- const key = blogImageKey(existing.id, encoded.sha256);- await putObject(key, encoded.bytes, "image/avif");+ const image = await storeBlogImage(existing.id, upload.bytes, kind);await del([uploadKey]);- const image = { key, url: blogImageUrl(key), width: encoded.width, height: encoded.height };if (kind === "inline") return NextResponse.json({ image });const row = await prisma.blogPost.update({where: { id: existing.id },- data: kind === "header" ? { headerKey: key } : { thumbnailKey: key },+ data: kind === "header" ? { headerKey: image.key } : { thumbnailKey: image.key },});revalidateBlogLater([row.slug]);return NextResponse.json({ image, post: adminPost(row) });diff --git a/site/src/app/su/blog/[id]/PostEditorPage.tsx b/site/src/app/su/blog/[id]/PostEditorPage.tsxindex 7eaa2180..9192b33c 100644--- a/site/src/app/su/blog/[id]/PostEditorPage.tsx+++ b/site/src/app/su/blog/[id]/PostEditorPage.tsx@@ -1,6 +1,6 @@"use client";-import { Ellipsis, ExternalLink, Eye, EyeOff, Save, Send } from "lucide-react";+import { Ellipsis, ExternalLink, Eye, EyeOff, Save, Send, Sparkles } from "lucide-react";import Link from "next/link";import { useCallback, useEffect, useMemo, useRef, useState } from "react";@@ -41,6 +41,7 @@ import {uploadBlogImage,useBlogPost,useBlogPosts,+ useGenerateBlogDetails,usePublishBlogPost,useSaveBlogPost,useUnpublishBlogPost,@@ -119,6 +120,7 @@ function PostEditor({ post }: { post: BlogPostAdminWithBody }) {const save = useSaveBlogPost(post.id);const publish = usePublishBlogPost(post.id);const unpublish = useUnpublishBlogPost(post.id);+ const generate = useGenerateBlogDetails(post.id);const [draft, setDraft] = useState<BlogPostInput>(() => inputOf(post));const [saved, setSaved] = useState<BlogPostInput>(() => inputOf(post));@@ -139,7 +141,7 @@ function PostEditor({ post }: { post: BlogPostAdminWithBody }) {useSuCrumb(draft.title.trim() || "Untitled post");const dirty = useMemo(() => JSON.stringify(draft) !== JSON.stringify(saved), [draft, saved]);- const busy = save.isPending || publish.isPending || unpublish.isPending;+ const busy = save.isPending || publish.isPending || unpublish.isPending || generate.isPending;const published = post.status === "PUBLISHED";const patch = (next: Partial<BlogPostInput>) => setDraft((current) => ({ ...current, ...next }));@@ -208,6 +210,16 @@ function PostEditor({ post }: { post: BlogPostAdminWithBody }) {unpublish.mutate(undefined, { onError: (error) => setNotice(error.message) });};+ // The copy lands in the form for a look before it is saved; the pictures+ // are on the row already and show through the query.+ const doGenerate = () => {+ setNotice(null);+ generate.mutate(+ { title: draft.title, body: draft.body },+ { onSuccess: ({ details }) => patch(details), onError: (error) => setNotice(error.message) },+ );+ };+const openPreview = async () => {if (dirty && !(await doSave())) return;window.open(`${SU_APP_ORIGIN}/api/blog/preview?id=${encodeURIComponent(post.id)}`, "_blank", "noopener");@@ -468,6 +480,16 @@ function PostEditor({ post }: { post: BlogPostAdminWithBody }) {) : null}</section></div>++ <div className="flex flex-wrap items-center justify-between gap-3 border-t pt-6">+ <p className="text-xs text-muted-foreground">+ {draft.body.trim() ? "Writes every detail and both pictures from the article. Change anything after." : "Write the article first."}+ </p>+ <Button variant="outline" disabled={busy || !draft.body.trim()} onClick={doGenerate}>+ <Sparkles />+ {generate.isPending ? "Generating…" : "Generate details"}+ </Button>+ </div></TabsContent></Tabs>);diff --git a/site/src/lib/blog/generate.ts b/site/src/lib/blog/generate.tsnew file mode 100644index 00000000..060b57a8--- /dev/null+++ b/site/src/lib/blog/generate.ts@@ -0,0 +1,151 @@+// One call fills the details of a post from its article: the search copy, the+// tags and keywords, the alt text, and a picture for the header and the+// thumbnail. The text model reads the article and answers with the copy and+// with a description of the picture; the image model paints that description.+// The copy comes back for the editor to look over; the picture is stored the+// way an upload is, so the boxes show it at once.+import { Modality, type GenerateContentParameters } from "@google/genai";+import { z } from "zod";++import {+ defaultGeminiClientFactory,+ geminiApiError,+ geminiCandidateParts,+ geminiCandidates,+ geminiClientConfig,+ stringValue,+} from "@/lib/inference/adapters/gemini-client";+import { geminiModelRoles } from "@/lib/inference/gemini-models";+import { isJsonObject, toJsonValue } from "@/lib/inference/json";+import { InferenceProviderError, type JsonValue } from "@/lib/inference/providers";+import { BLOG_DESCRIPTION_MAX, BLOG_DESCRIPTION_MIN, BLOG_SEO_TITLE_MAX } from "@/lib/blog/schema";++export const TEXT_MODEL = geminiModelRoles.chat;+export const IMAGE_MODEL = geminiModelRoles.imageGeneration;++const SITE_SUFFIX = " | Donkey Cut";++export const blogDetailsSchema = z+ .object({+ seoTitle: z.string().trim().max(200),+ excerpt: z.string().trim().max(400),+ summary: z.string().trim().max(400),+ tags: z.array(z.string().trim().min(1).max(80)).max(30),+ keywords: z.array(z.string().trim().min(1).max(80)).max(30),+ headerAlt: z.string().trim().max(400),+ imagePrompt: z.string().trim().min(1).max(4000),+ })+ .strict();++export type BlogDetails = z.infer<typeof blogDetailsSchema>;++const detailsPrompt = (title: string, article: string) =>+ [+ "You write the details of a post on the Donkey Cut blog. Donkey Cut is a video editor that runs in the browser; its readers edit video and want to do it well.",+ "Read the article and answer with JSON of this exact shape, nothing else:",+ JSON.stringify({+ seoTitle: "string",+ excerpt: "string",+ summary: "string",+ tags: ["string"],+ keywords: ["string"],+ headerAlt: "string",+ imagePrompt: "string",+ }),+ `seoTitle: the title as it appears in a search result, at most ${BLOG_SEO_TITLE_MAX - SITE_SUFFIX.length} characters, leading with the words a reader searches for. The site name is added after it.`,+ `excerpt: the meta description under the search result, ${BLOG_DESCRIPTION_MIN} to ${BLOG_DESCRIPTION_MAX} characters, one or two plain sentences saying what the reader gets.`,+ `summary: the answer-first line shown under the title on the page, ${BLOG_DESCRIPTION_MIN} to ${BLOG_DESCRIPTION_MAX} characters, stating the article's point outright.`,+ "tags: three to five lowercase topic words or short phrases the post files under.",+ "keywords: five to ten search phrases the article answers, most specific first.",+ "headerAlt: one sentence describing the header picture for a reader who cannot see it.",+ "imagePrompt: a description of one picture that stands for the article, for an image model: the subject, the scene, the framing. No text, letters, logos or user interface in the picture.",+ "Write plainly. No marketing words, no exclamation marks, no claims the article does not make.",+ `Title: ${title.trim() || "(untitled)"}`,+ "Article:",+ article,+ ].join("\n\n");++const IMAGE_STYLE =+ "Editorial illustration for a blog post, flat shapes, bold ink outlines, a warm cream background, two or three accent colours, generous negative space. No text, letters, logos or user interface.";++function readJson(text: string): unknown {+ const cleaned = text+ .trim()+ .replace(/^```(?:json)?\s*/i, "")+ .replace(/```\s*$/, "")+ .trim();+ try {+ return JSON.parse(cleaned);+ } catch {+ const start = cleaned.indexOf("{");+ const end = cleaned.lastIndexOf("}");+ if (start < 0 || end <= start) return null;+ try {+ return JSON.parse(cleaned.slice(start, end + 1));+ } catch {+ return null;+ }+ }+}++function client() {+ const config = geminiClientConfig(process.env);+ if (!config.configured) {+ throw new InferenceProviderError("Generation is not configured on this deployment.", { statusCode: 500 });+ }+ return defaultGeminiClientFactory(config.options);+}++export type Generated<T> = { value: T; usage: JsonValue };++// The copy and the picture description, read off the article.+export async function generateBlogDetails(title: string, article: string): Promise<Generated<BlogDetails>> {+ let raw: unknown;+ try {+ raw = await client().models.generateContent({+ model: TEXT_MODEL,+ contents: [{ role: "user", parts: [{ text: detailsPrompt(title, article) }] }],+ // JSON mode without a schema; constrained decoding degrades the copy.+ config: { responseMimeType: "application/json" },+ });+ } catch (error) {+ throw geminiApiError("The writing model is unavailable.", error);+ }+ const json = toJsonValue(raw);+ const text = geminiCandidateParts(geminiCandidates(json)[0])+ .map((part) => stringValue(part.text) ?? "")+ .join("");+ const parsed = blogDetailsSchema.safeParse(readJson(text));+ if (!parsed.success) {+ throw new InferenceProviderError("The writing model returned an unreadable answer.", { statusCode: 502 });+ }+ const usage = isJsonObject(json) ? (json.usageMetadata ?? null) : null;+ return { value: parsed.data, usage };+}++// One picture, wide enough for the header; the thumbnail is a crop of it.+export async function generateBlogPicture(imagePrompt: string): Promise<Generated<Buffer>> {+ const params: GenerateContentParameters = {+ model: IMAGE_MODEL,+ contents: [{ role: "user", parts: [{ text: `${imagePrompt}\n\n${IMAGE_STYLE}` }] }],+ config: {+ responseModalities: [Modality.IMAGE, Modality.TEXT],+ imageConfig: { aspectRatio: "16:9", imageSize: "2K" },+ },+ };+ let raw: unknown;+ try {+ raw = await client().models.generateContent(params);+ } catch (error) {+ throw geminiApiError("The image model is unavailable.", error);+ }+ const json = toJsonValue(raw);+ for (const candidate of geminiCandidates(json)) {+ for (const part of geminiCandidateParts(candidate)) {+ const inline = part.inlineData ?? part.inline_data;+ const data = isJsonObject(inline) ? stringValue(inline.data) : undefined;+ if (data) return { value: Buffer.from(data, "base64"), usage: { generationCount: 1 } };+ }+ }+ throw new InferenceProviderError("The image model returned no picture.", { statusCode: 502 });+}diff --git a/site/src/lib/blog/images.ts b/site/src/lib/blog/images.tsindex e1e02f3b..18c30b0d 100644--- a/site/src/lib/blog/images.ts+++ b/site/src/lib/blog/images.ts@@ -4,6 +4,9 @@import { createHash } from "node:crypto";import sharp from "sharp";+import { putObject } from "@/cut/server/cloud/r2";+import { blogImageKey, blogImageUrl } from "@/lib/blog/keys";+export type BlogImageKind = "header" | "thumbnail" | "inline";export const BLOG_IMAGE_SIZES: Record<BlogImageKind, { width: number; height?: number }> = {@@ -35,3 +38,13 @@ export async function encodeBlogImage(source: Buffer, kind: BlogImageKind): Promsha256: createHash("sha256").update(data).digest("hex"),};}++export type StoredBlogImage = { key: string; url: string; width: number; height: number };++// Encodes and writes the image under its own hash; the caller sets it on the row.+export async function storeBlogImage(postId: string, source: Buffer, kind: BlogImageKind): Promise<StoredBlogImage> {+ const encoded = await encodeBlogImage(source, kind);+ const key = blogImageKey(postId, encoded.sha256);+ await putObject(key, encoded.bytes, "image/avif");+ return { key, url: blogImageUrl(key), width: encoded.width, height: encoded.height };+}diff --git a/site/src/queries/blog.ts b/site/src/queries/blog.tsindex b7a02dd0..e936b1d0 100644--- a/site/src/queries/blog.ts+++ b/site/src/queries/blog.ts@@ -76,6 +76,25 @@ export function useUnpublishBlogPost(id: string) {});}+export type BlogGeneratedDetails = Pick<+ BlogPostInput,+ "seoTitle" | "excerpt" | "summary" | "tags" | "keywords" | "headerAlt"+>;++// Details written off the article. The pictures land on the row at once; the+// copy is the caller's to put in the form.+export function useGenerateBlogDetails(id: string) {+ const takePost = useTakePost();+ return useMutation({+ mutationFn: (input: { title: string; body: string }) =>+ apiFetch<{ post: BlogPostAdmin; details: BlogGeneratedDetails }>(postUrl(id, "/generate"), {+ body: JSON.stringify(input),+ method: "POST",+ }),+ onSuccess: ({ post }) => takePost(post),+ });+}+export function useDeleteBlogPost() {const queryClient = useQueryClient();return useMutation({