Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.
TypeScript accessibilityalternativecloudflarecloudflare-workersdesignfreefrontendopen-sourceradix-uireactremixweb-developmentwebflowwebsite-builderwebsite-developmentwebsiteswebstudio
feat: search images in pexels with ai (#2453)
Bogdan Chadkin81e58fe1f513
diff --git a/apps/builder/app/env/env.server.ts b/apps/builder/app/env/env.server.tsindex 6982a2b5af..2b6037e99d 100644--- a/apps/builder/app/env/env.server.ts+++ b/apps/builder/app/env/env.server.ts@@ -96,6 +96,8 @@ const env = {*/OPENAI_KEY: process.env.OPENAI_KEY,OPENAI_ORG: process.env.OPENAI_ORG,++ PEXELS_API_KEY: process.env.PEXELS_API_KEY,};export type ServerEnv = typeof env;diff --git a/apps/builder/app/routes/rest.ai.ts b/apps/builder/app/routes/rest.ai.tsindex 6797be20c5..e641890e9d 100644--- a/apps/builder/app/routes/rest.ai.ts+++ b/apps/builder/app/routes/rest.ai.ts@@ -12,6 +12,7 @@ import {import {copywriter as clientCopywriter,operations as clientOperations,+ queryImagesAndMutateTemplate,} from "@webstudio-is/ai";import { isFeatureEnabled } from "@webstudio-is/feature-flags";import env from "~/env/env.server";@@ -64,6 +65,19 @@ export const action = async ({ request }: ActionArgs) => {};}+ if (env.PEXELS_API_KEY === undefined) {+ return {+ id: "ai",+ ...createErrorResponse({+ error: "ai.invalidApiKey",+ status: 401,+ debug: "Invalid Pexels api key",+ }),+ llmMessages: [],+ };+ }+ const PEXELS_API_KEY = env.PEXELS_API_KEY;+if (env.OPENAI_ORG === undefined ||env.OPENAI_ORG.startsWith("org-") === false@@ -192,9 +206,16 @@ export const action = async ({ request }: ActionArgs) => {// and then handle the generation request with a standalone chain called template-generator// that has a dedicate and comprehensive prompt.- const generateTemplatePrompts = response.data.filter(- (operation) => operation.operation === "generateTemplatePrompt"- ) as operations.generateTemplatePrompt.wsOperation[];+ const generateTemplatePrompts: {+ dataIndex: number;+ operation: operations.generateTemplatePrompt.wsOperation;+ }[] = [];+ response.data.forEach((operation, dataIndex) => {+ if (operation.operation === "generateTemplatePrompt") {+ // preserve the index in response.data to update it after executing operations+ generateTemplatePrompts.push({ dataIndex, operation });+ }+ });if (generateTemplatePrompts.length > 0) {const generationModel = createGptModel({@@ -208,18 +229,29 @@ export const action = async ({ request }: ActionArgs) => {templateGenerator.createChain<GptModelMessageFormat>();const results = await Promise.all(- generateTemplatePrompts.map((operation, index) =>- generationChain({+ generateTemplatePrompts.map(async ({ dataIndex, operation }) => {+ const result = await generationChain({model: generationModel,context: {prompt: operation.llmPrompt,components,},- }).then((result) => [index, result] as const)- )+ });+ if (result.success) {+ await queryImagesAndMutateTemplate({+ template: result.data,+ apiKey: PEXELS_API_KEY,+ });+ }+ return {+ dataIndex,+ operation,+ result,+ };+ }));- for (const [index, result] of results) {+ for (const { dataIndex, operation, result } of results) {llmMessages.push(...result.llmMessages);if (result.success === false) {@@ -230,11 +262,10 @@ export const action = async ({ request }: ActionArgs) => {}// Replace generateTemplatePrompt.wsOperation with the AI-generated Webstudio template.- const generateTemplatePrompt = generateTemplatePrompts[index];- response.data[index] = {+ response.data[dataIndex] = {operation: "insertTemplate",- addTo: generateTemplatePrompt.addTo,- addAtIndex: generateTemplatePrompt.addAtIndex,+ addTo: operation.addTo,+ addAtIndex: operation.addAtIndex,template: result.data,};}diff --git a/packages/ai/src/image.ts b/packages/ai/src/image.tsnew file mode 100644index 0000000000..86f0434618--- /dev/null+++ b/packages/ai/src/image.ts@@ -0,0 +1,155 @@+import { traverseTemplate } from "@webstudio-is/jsx-utils";+import type {+ EmbedTemplateInstance,+ EmbedTemplateProp,+ WsEmbedTemplate,+} from "@webstudio-is/react-sdk";++const isRemoteImageGeneratedByAi = (url: string) => {+ // wsai search param is added when image is queried by ai+ // and allows to distinct image urls added by user manually+ try {+ return new URL(url).searchParams.get("wsai") === "true";+ } catch {+ return false;+ }+};++const canAiRegenerateImage = (src: undefined | EmbedTemplateProp) => {+ // asset images should not be regenerated+ if (src && src?.type !== "string") {+ return false;+ }+ // regenerate when image is not specified+ if (src === undefined || src.value.trim() === "") {+ return true;+ }+ return isRemoteImageGeneratedByAi(src.value);+};++type PexelsPhotoResource = {+ id: number;+ width: number;+ height: number;+ url: string;+ photographer: string;+ photographer_url: string;+ photographer_id: number;+ avg_color: string;+ src: {+ original: string;+ large2x: string;+ large: string;+ medium: string;+ small: string;+ portrait: string;+ landscape: string;+ tiny: string;+ };+ liked: boolean;+ alt: string;+};++type PexelsSearchResponse = {+ photos: PexelsPhotoResource[];+ page: number;+ per_page: number;+ total_results: number;+ next_page?: string;+ prev_page?: string;+};++const searchImageInPexels = async ({+ query,+ apiKey,+}: {+ query: string;+ apiKey: string;+}) => {+ const url = new URL("https://api.pexels.com/v1/search");+ url.searchParams.set("query", query);+ url.searchParams.set("per_page", "1");+ const response = await fetch(url, {+ headers: {+ Authorization: apiKey,+ },+ });+ const result: PexelsSearchResponse = await response.json();+ return result;+};++const queryImageAndMutateInstance = async (+ apiKey: string,+ instance: EmbedTemplateInstance+) => {+ if (instance.props === undefined) {+ return;+ }+ const alt = instance.props.find((prop) => prop.name === "alt");+ if (alt?.type !== "string") {+ return;+ }+ const imageSearchResult = await searchImageInPexels({+ query: alt.value,+ apiKey,+ });+ const [result] = imageSearchResult.photos;+ const url = new URL(result.src.original);+ // mark remote image as generated by ai+ url.searchParams.set("wsai", "true");+ const newSrc: EmbedTemplateProp = {+ name: "src",+ type: "string",+ value: url.href,+ };+ // @todo simplify with props and styles object format+ const srcIndex = instance.props.findIndex((prop) => prop.name === "src");+ if (srcIndex === -1) {+ instance.props.push(newSrc);+ } else {+ instance.props[srcIndex] = newSrc;+ }+ // prevent image deformation when ai specifies image size+ const hasObjectFit = instance.styles?.some(+ (styleDecl) => styleDecl.property === "objectFit"+ );+ if (hasObjectFit === false) {+ instance.styles = instance.styles ?? [];+ instance.styles.push({+ property: "objectFit",+ value: { type: "keyword", value: "cover" },+ });+ }+};++export const queryImagesAndMutateTemplate = async ({+ template,+ apiKey,+}: {+ template: WsEmbedTemplate;+ apiKey: string;+}) => {+ const imageInstances = new Set<EmbedTemplateInstance>();+ traverseTemplate(template, (instance) => {+ if (instance.type === "instance" && instance.component === "Image") {+ if (instance.props === undefined) {+ return;+ }+ const src = instance.props.find((prop) => prop.name === "src");+ const alt = instance.props.find((prop) => prop.name === "alt");+ if (+ canAiRegenerateImage(src) === false ||+ // skip when no alt to generate image from+ alt?.type !== "string"+ ) {+ return;+ }+ imageInstances.add(instance);+ }+ });+ await Promise.allSettled(+ Array.from(imageInstances).map((instance) =>+ queryImageAndMutateInstance(apiKey, instance)+ )+ );+};diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.tsindex 0249e8c629..c8beb55525 100644--- a/packages/ai/src/index.ts+++ b/packages/ai/src/index.ts@@ -11,3 +11,5 @@ export * from "./utils/remix-streaming-text-response";/** Types */export * from "./types";++export * from "./image";