Skip to content

The Open Standard for Generative UI

Streaming helpers for integration with crayon (#37)

Rabi Shanker Guhaf4f2283119ee
diff --git a/docs/docs/quickstart/002-nextjs.mdx b/docs/docs/quickstart/002-nextjs.mdx
index 8ebeea284..ad608b0af 100644
--- a/docs/docs/quickstart/002-nextjs.mdx
+++ b/docs/docs/quickstart/002-nextjs.mdx
@@ -29,17 +29,17 @@ import TabItem from '@theme/TabItem';
<Tabs groupId="js-package-manager">
<TabItem value="npm" label="npm">
```bash
-npm install @crayonai/react-ui @crayonai/react-core
+npm install @crayonai/react-ui @crayonai/react-core @crayonai/stream
```
</TabItem>
<TabItem value="pnpm" label="pnpm">
```bash
-pnpm add @crayonai/crayon-ui @crayonai/react-core
+pnpm add @crayonai/crayon-ui @crayonai/react-core @crayonai/stream
```
</TabItem>
<TabItem value="yarn" label="yarn">
```bash
-yarn add @crayonai/crayon-ui @crayonai/react-core
+yarn add @crayonai/crayon-ui @crayonai/react-core @crayonai/stream
```
</TabItem>
</Tabs>
@@ -48,13 +48,36 @@ yarn add @crayonai/crayon-ui @crayonai/react-core
## Add the CrayonChat component to your app
```tsx title="page.tsx"
+"use client";
+
+import type { Message } from "@crayonai/react-core";
import { CrayonChat } from "@crayonai/react-ui";
import "@crayonai/react-ui/styles/index.css";
+const processMessage = async ({
+ threadId,
+ messages,
+ abortController,
+}: {
+ threadId: string;
+ messages: Message[];
+ abortController: AbortController;
+}) => {
+ const response = await fetch("/api/chat", {
+ method: "POST",
+ body: JSON.stringify({ threadId, messages }),
+ headers: {
+ "Content-Type": "application/json",
+ Accept: "text/event-stream",
+ },
+ signal: abortController.signal,
+ });
+ return response;
+};
+
export default function Home() {
- return <CrayonChat agentName="Crayon" logoUrl="" />;
+ return <CrayonChat processMessage={processMessage} />;
}
-
```
## Integrate LLM with your backend route
@@ -62,23 +85,41 @@ export default function Home() {
```tsx title="api/chat/route.ts"
import { NextRequest, NextResponse } from "next/server";
import OpenAI from "openai";
+import {
+ fromOpenAICompletion,
+ TextResponseSchema,
+ toOpenAIMessages,
+} from "@crayonai/stream";
+import { Message } from "@crayonai/react-core";
+
+const TemplatesJsonSchema = {
+ type: "object",
+ properties: {
+ response: {
+ type: "array",
+ items: {
+ oneOf: [TextResponseSchema],
+ },
+ },
+ },
+} as const;
export async function POST(req: NextRequest) {
- const { messages } = await req.json();
+ const { messages } = (await req.json()) as { messages: Message[] };
const client = new OpenAI();
const llmStream = await client.chat.completions.create({
model: "gpt-4o",
- messages: messages,
+ messages: toOpenAIMessages(messages),
stream: true,
- });
- const responseStream = new ReadableStream({
- start: async (controller) => {
- for await (const chunk of llmStream) {
- controller.enqueue(chunk.choices[0].delta.content);
- }
+ response_format: {
+ type: "json_schema",
+ json_schema: {
+ name: "json_schema",
+ schema: TemplatesJsonSchema,
+ },
},
});
-
+ const responseStream = fromOpenAICompletion(llmStream);
return new NextResponse(responseStream, {
headers: {
"Content-Type": "text/event-stream",
diff --git a/docs/static/img/next.svg b/docs/static/img/next.svg
deleted file mode 100644
index 5174b28c5..000000000
--- a/docs/static/img/next.svg
+++ /dev/null
@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 394 80"><path fill="#000" d="M262 0h68.5v12.7h-27.2v66.6h-13.6V12.7H262V0ZM149 0v12.7H94v20.4h44.3v12.6H94v21h55v12.6H80.5V0h68.7zm34.3 0h-17.8l63.8 79.4h17.9l-32-39.7 32-39.6h-17.9l-23 28.6-23-28.6zm18.3 56.7-9-11-27.1 33.7h17.8l18.3-22.7z"/><path fill="#000" d="M81 79.3 17 0H0v79.3h13.6V17l50.2 62.3H81Zm252.6-.4c-1 0-1.8-.4-2.5-1s-1.1-1.6-1.1-2.6.3-1.8 1-2.5 1.6-1 2.6-1 1.8.3 2.5 1a3.4 3.4 0 0 1 .6 4.3 3.7 3.7 0 0 1-3 1.8zm23.2-33.5h6v23.3c0 2.1-.4 4-1.3 5.5a9.1 9.1 0 0 1-3.8 3.5c-1.6.8-3.5 1.3-5.7 1.3-2 0-3.7-.4-5.3-1s-2.8-1.8-3.7-3.2c-.9-1.3-1.4-3-1.4-5h6c.1.8.3 1.6.7 2.2s1 1.2 1.6 1.5c.7.4 1.5.5 2.4.5 1 0 1.8-.2 2.4-.6a4 4 0 0 0 1.6-1.8c.3-.8.5-1.8.5-3V45.5zm30.9 9.1a4.4 4.4 0 0 0-2-3.3 7.5 7.5 0 0 0-4.3-1.1c-1.3 0-2.4.2-3.3.5-.9.4-1.6 1-2 1.6a3.5 3.5 0 0 0-.3 4c.3.5.7.9 1.3 1.2l1.8 1 2 .5 3.2.8c1.3.3 2.5.7 3.7 1.2a13 13 0 0 1 3.2 1.8 8.1 8.1 0 0 1 3 6.5c0 2-.5 3.7-1.5 5.1a10 10 0 0 1-4.4 3.5c-1.8.8-4.1 1.2-6.8 1.2-2.6 0-4.9-.4-6.8-1.2-2-.8-3.4-2-4.5-3.5a10 10 0 0 1-1.7-5.6h6a5 5 0 0 0 3.5 4.6c1 .4 2.2.6 3.4.6 1.3 0 2.5-.2 3.5-.6 1-.4 1.8-1 2.4-1.7a4 4 0 0 0 .8-2.4c0-.9-.2-1.6-.7-2.2a11 11 0 0 0-2.1-1.4l-3.2-1-3.8-1c-2.8-.7-5-1.7-6.6-3.2a7.2 7.2 0 0 1-2.4-5.7 8 8 0 0 1 1.7-5 10 10 0 0 1 4.3-3.5c2-.8 4-1.2 6.4-1.2 2.3 0 4.4.4 6.2 1.2 1.8.8 3.2 2 4.3 3.4 1 1.4 1.5 3 1.5 5h-5.8z"/></svg>
\ No newline at end of file
diff --git a/docs/static/img/vercel.svg b/docs/static/img/vercel.svg
deleted file mode 100644
index 770539603..000000000
--- a/docs/static/img/vercel.svg
+++ /dev/null
@@ -1 +0,0 @@
-<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1155 1000"><path d="m577.3 0 577.4 1000H0z" fill="#fff"/></svg>
\ No newline at end of file
diff --git a/js/packages/react-core/package.json b/js/packages/react-core/package.json
index 3bcb582e0..1b52b5829 100644
--- a/js/packages/react-core/package.json
+++ b/js/packages/react-core/package.json
@@ -2,6 +2,7 @@
"name": "@crayonai/react-core",
"version": "0.5.1",
"description": "Generative UI SDK",
+ "license": "MIT",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"files": [
@@ -38,8 +39,7 @@
"prettier-plugin-organize-imports": "^3.2.4"
},
"keywords": [],
- "author": "",
- "license": "ISC",
+ "author": "[email protected]",
"dependencies": {
"zod": "^3.24.1"
}
diff --git a/js/packages/react-core/src/types/message.ts b/js/packages/react-core/src/types/message.ts
index e4014cbfc..9e9829e29 100644
--- a/js/packages/react-core/src/types/message.ts
+++ b/js/packages/react-core/src/types/message.ts
@@ -39,4 +39,4 @@ export type Message = UserMessage | AssistantMessage;
* Represents a message being created, with an optional ID
* @extends Omit<Message, "id">
*/
-export type CreateMessage = Omit<Message, "id">;
+export type CreateMessage = Omit<UserMessage, "id">;
diff --git a/js/packages/react-ui/package.json b/js/packages/react-ui/package.json
index b9e5ae7f8..9e3cb3fae 100644
--- a/js/packages/react-ui/package.json
+++ b/js/packages/react-ui/package.json
@@ -1,6 +1,7 @@
{
"type": "module",
"name": "@crayonai/react-ui",
+ "license": "MIT",
"version": "0.5.1",
"description": "Component library for Generative UI SDK",
"main": "dist/index.js",
@@ -110,8 +111,7 @@
"vite": "^5.0.0"
},
"keywords": [],
- "author": "",
- "license": "ISC",
+ "author": "[email protected]",
"eslintConfig": {
"extends": [
"plugin:storybook/recommended"
diff --git a/js/packages/react-ui/src/components/CrayonChat/CrayonChat.tsx b/js/packages/react-ui/src/components/CrayonChat/CrayonChat.tsx
index cb7187993..e048bcd77 100644
--- a/js/packages/react-ui/src/components/CrayonChat/CrayonChat.tsx
+++ b/js/packages/react-ui/src/components/CrayonChat/CrayonChat.tsx
@@ -30,14 +30,14 @@ import { ThemeProvider } from "../ThemeProvider";
interface CrayonChatProps {
processMessage?: (params: {
threadId: string;
- message: CreateMessage;
+ messages: Message[];
abortController: AbortController;
}) => Promise<Response>;
createThread?: (message: CreateMessage) => Promise<Thread>;
threadManager?: ThreadManager;
threadListManager?: ThreadListManager;
- logoUrl: string;
- agentName: string;
+ logoUrl?: string;
+ agentName?: string;
responseTemplates?: ResponseTemplate[];
}
@@ -45,8 +45,8 @@ export const CrayonChat = ({
processMessage,
threadManager: userThreadManager,
threadListManager: userThreadListManager,
- logoUrl,
- agentName,
+ logoUrl = "https://crayonai.org/img/logo.png",
+ agentName = "My Agent",
responseTemplates,
createThread,
}: CrayonChatProps) => {
@@ -81,12 +81,13 @@ export const CrayonChat = ({
return Promise.resolve(messages);
},
onProcessMessage: async ({ message, abortController, threadManager }) => {
- threadManager.appendMessages({
+ const newMessage: UserMessage = {
id: crypto.randomUUID(),
role: "user",
type: "prompt",
- message: message.message as string,
- });
+ message: message.message,
+ };
+ threadManager.appendMessages(newMessage);
let threadId = threadListManager.selectedThreadId;
if (!threadId) {
@@ -97,7 +98,11 @@ export const CrayonChat = ({
invariant(processMessage, "processMessage is required");
- const response = await processMessage({ threadId, message, abortController });
+ const response = await processMessage({
+ threadId,
+ messages: [...threadManager.messages, newMessage],
+ abortController,
+ });
await processStreamedMessage({
response,
createMessage: threadManager.appendMessages,
diff --git a/js/packages/react-ui/src/components/Shell/Thread.tsx b/js/packages/react-ui/src/components/Shell/Thread.tsx
index eba430f29..d7b70a7b0 100644
--- a/js/packages/react-ui/src/components/Shell/Thread.tsx
+++ b/js/packages/react-ui/src/components/Shell/Thread.tsx
@@ -157,6 +157,7 @@ export const Composer = ({ className }: { className?: string }) => {
}
processMessage({
+ type: "prompt",
role: "user",
message: textContent,
});
diff --git a/js/packages/stream/README.md b/js/packages/stream/README.md
new file mode 100644
index 000000000..9ebe9fc8e
--- /dev/null
+++ b/js/packages/stream/README.md
@@ -0,0 +1 @@
+## Streaming Helpers
diff --git a/js/packages/stream/package.json b/js/packages/stream/package.json
new file mode 100644
index 000000000..485c97c32
--- /dev/null
+++ b/js/packages/stream/package.json
@@ -0,0 +1,40 @@
+{
+ "name": "@crayonai/stream",
+ "version": "0.5.0",
+ "description": "Generative UI SDK",
+ "main": "dist/index.js",
+ "types": "dist/index.d.ts",
+ "files": [
+ "dist"
+ ],
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1",
+ "build": "tsc -p .",
+ "lint:check": "eslint ./src",
+ "lint:fix": "eslint ./src --fix",
+ "format:fix": "prettier --write ./src",
+ "format:check": "prettier --check ./src",
+ "prepare": "pnpm run build",
+ "ci": "pnpm run lint:check && pnpm run format:check"
+ },
+ "devDependencies": {
+ "@types/node": "^22.12.0",
+ "@typescript-eslint/eslint-plugin": "^8.18.0",
+ "eslint": "^9.17.0",
+ "eslint-config-prettier": "^9.1.0",
+ "eslint-plugin-prettier": "^5.2.1",
+ "eslint-plugin-react-hooks": "^5.1.0",
+ "eslint-plugin-react-refresh": "^0.4.16",
+ "eslint-plugin-unused-imports": "^3.1.0",
+ "prettier-plugin-organize-imports": "^3.2.4"
+ },
+ "keywords": [],
+ "author": "",
+ "license": "ISC",
+ "peerDependencies": {
+ "@crayonai/react-core": "workspace:*"
+ },
+ "dependencies": {
+ "best-effort-json-parser": "^1.1.2"
+ }
+}
diff --git a/js/packages/stream/src/index.ts b/js/packages/stream/src/index.ts
new file mode 100644
index 000000000..243e828ee
--- /dev/null
+++ b/js/packages/stream/src/index.ts
@@ -0,0 +1,4 @@
+export * from "./openai";
+export * from "./textSchema";
+export * from "./transformer";
+export * from "./types";
diff --git a/js/packages/stream/src/openai.ts b/js/packages/stream/src/openai.ts
new file mode 100644
index 000000000..b86905675
--- /dev/null
+++ b/js/packages/stream/src/openai.ts
@@ -0,0 +1,64 @@
+import type { Message } from "@crayonai/react-core";
+import { CrayonDataStreamTransformer, TransformerOpts } from "./transformer";
+// These types are defined here so as to not introduce a dependency
+// on openai library directly.
+interface ChatCompletionChunk {
+ choices: Array<{
+ delta?: {
+ content?: string | null;
+ };
+ }>;
+}
+interface OpenAIMessage {
+ role: string;
+ content: string | { type: "text"; text: string }[];
+}
+
+type ChatCompletionStreamingRunner = AsyncIterable<ChatCompletionChunk>;
+
+export const fromOpenAICompletion = (
+ completion: ChatCompletionStreamingRunner,
+ opts?: TransformerOpts,
+) => {
+ const readableStream = new ReadableStream({
+ async start(controller) {
+ try {
+ for await (const chunk of completion) {
+ const content = chunk.choices[0]?.delta?.content;
+ if (content) {
+ controller.enqueue(content);
+ }
+ }
+ controller.close();
+ } catch (error) {
+ controller.error(error);
+ }
+ },
+ });
+ return readableStream.pipeThrough(new CrayonDataStreamTransformer(opts));
+};
+
+export const toOpenAIMessages = (messages: Message[]) => {
+ const openAIMessages: OpenAIMessage[] = [];
+ for (const message of messages) {
+ if (!message.message) {
+ continue;
+ }
+ if (typeof message.message === "string") {
+ openAIMessages.push({
+ role: message.role,
+ content: message.message,
+ });
+ } else if (Array.isArray(message.message)) {
+ openAIMessages.push({
+ role: message.role,
+ content: message.message.map((m) =>
+ typeof m === "string"
+ ? { type: "text", text: m }
+ : { type: "text", text: JSON.stringify(m) },
+ ),
+ });
+ }
+ }
+ return openAIMessages;
+};
diff --git a/js/packages/stream/src/textSchema.ts b/js/packages/stream/src/textSchema.ts
new file mode 100644
index 000000000..787943df5
--- /dev/null
+++ b/js/packages/stream/src/textSchema.ts
@@ -0,0 +1,15 @@
+// OpenAPI object definition for a simple text based reponse.
+// This should be used as the default/fallback response type when
+// using structured outputs.
+export const TextResponseSchema = {
+ type: "object",
+ properties: {
+ type: { const: "text" },
+ text: {
+ type: "string",
+ description: "text message to be displayed to the user",
+ },
+ },
+ required: ["type", "text"],
+ additionalProperties: false,
+};
diff --git a/js/packages/stream/src/transformer.ts b/js/packages/stream/src/transformer.ts
new file mode 100644
index 000000000..093247c52
--- /dev/null
+++ b/js/packages/stream/src/transformer.ts
@@ -0,0 +1,72 @@
+import { parse } from "best-effort-json-parser";
+import { ResponseTemplateChunk, TextChunk } from "./types";
+// Define the types for the data flowing through the stream
+type InputType = string; // Change this to match your input type
+type OutputType = string; // Change this to match your output type
+
+export interface TransformerOpts {
+ onFinish: (controller: TransformStreamDefaultController<OutputType>) => Promise<void>;
+}
+
+export class CrayonDataStreamTransformer implements TransformStream<InputType, OutputType> {
+ readonly readable: ReadableStream<OutputType>;
+ readonly writable: WritableStream<InputType>;
+
+ constructor(opts?: TransformerOpts) {
+ let streamedContent = "";
+ const transform = new TransformStream({
+ transform: async (
+ content: InputType,
+ controller: TransformStreamDefaultController<OutputType>,
+ ) => {
+ try {
+ const previousParsed = parse(streamedContent);
+ streamedContent += content;
+ const parsed = parse(streamedContent);
+
+ if (previousParsed.response && parsed.response) {
+ if (previousParsed.response.length === parsed.response.length) {
+ const newContent = parsed.response.pop();
+ const lastContent = previousParsed.response.pop();
+ if (newContent.type === "text" && newContent.text) {
+ const textPart = newContent.text.substring(lastContent.text?.length);
+ if (textPart.length > 0) {
+ controller.enqueue(new TextChunk(textPart).toSSEString());
+ }
+ }
+ } else {
+ const lastTemplate = previousParsed.response.pop();
+ if (typeof lastTemplate === "object") {
+ controller.enqueue(new ResponseTemplateChunk(lastTemplate).toSSEString());
+ }
+
+ const newContent = parsed.response.pop();
+
+ if (newContent.type === "text" && newContent.text) {
+ controller.enqueue(new TextChunk(newContent).toSSEString());
+ }
+ }
+ }
+ } catch (error) {
+ controller.error(error);
+ }
+ },
+
+ flush: async (controller: TransformStreamDefaultController<OutputType>) => {
+ const parsed = parse(streamedContent);
+ if (
+ parsed.response &&
+ parsed.response.length &&
+ parsed.response[parsed.response.length - 1]?.type !== "text"
+ ) {
+ const lastTemplate = parsed.response.pop();
+ controller.enqueue(new ResponseTemplateChunk(lastTemplate).toSSEString());
+ }
+ await opts?.onFinish(controller);
+ },
+ });
+
+ this.readable = transform.readable;
+ this.writable = transform.writable;
+ }
+}
diff --git a/js/packages/stream/src/types.ts b/js/packages/stream/src/types.ts
new file mode 100644
index 000000000..0dce93690
--- /dev/null
+++ b/js/packages/stream/src/types.ts
@@ -0,0 +1,28 @@
+const escapeString = (s: string) => {
+ return s.replaceAll('"', '\\"').replaceAll("\n", "\\n");
+};
+
+interface Chunk {
+ toSSEString(): string;
+}
+
+export class TextChunk implements Chunk {
+ constructor(private readonly data: string) {}
+
+ toSSEString(): string {
+ return `0:${escapeString(this.data)}\n`;
+ }
+}
+
+export interface ResponseTemplate {
+ name: string;
+ templateProps: object;
+}
+
+export class ResponseTemplateChunk implements Chunk {
+ constructor(private readonly template: ResponseTemplate) {}
+
+ toSSEString(): string {
+ return `1:${escapeString(JSON.stringify(this.template))}\n`;
+ }
+}
diff --git a/js/packages/stream/tsconfig.json b/js/packages/stream/tsconfig.json
new file mode 100644
index 000000000..fb393558a
--- /dev/null
+++ b/js/packages/stream/tsconfig.json
@@ -0,0 +1,17 @@
+{
+ "$schema": "https://json.schemastore.org/tsconfig",
+ "extends": "../../tsconfig.json",
+ "include": ["src/**/*.ts"],
+ "compilerOptions": {
+ "target": "es2021",
+ "lib": ["es2021"],
+ "module": "commonjs",
+ "moduleResolution": "node",
+ "outDir": "./dist",
+ "rootDir": "./src",
+ "paths": {
+ "@/*": ["./src/*"],
+ },
+ "types": ["node"]
+ }
+}
diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml
index e8c1f687e..3fa53799c 100644
--- a/js/pnpm-lock.yaml
+++ b/js/pnpm-lock.yaml
@@ -58,7 +58,7 @@ importers:
version: 2.6.0
tailwindcss-animate:
specifier: ^1.0.7
- version: 1.0.7
+ version: 1.0.7([email protected])
tiny-invariant:
specifier: ^1.3.3
version: 1.3.3
@@ -275,6 +275,43 @@ importers:
specifier: ^5.0.0
+ packages/stream:
+ dependencies:
+ '@crayonai/react-core':
+ specifier: workspace:*
+ version: link:../react-core
+ best-effort-json-parser:
+ specifier: ^1.1.2
+ version: 1.1.2
+ devDependencies:
+ '@types/node':
+ specifier: ^22.12.0
+ version: 22.13.1
+ '@typescript-eslint/eslint-plugin':
+ specifier: ^8.18.0
+ eslint:
+ specifier: ^9.17.0
+ version: 9.19.0([email protected])
+ eslint-config-prettier:
+ specifier: ^9.1.0
+ eslint-plugin-prettier:
+ specifier: ^5.2.1
+ eslint-plugin-react-hooks:
+ specifier: ^5.1.0
+ eslint-plugin-react-refresh:
+ specifier: ^0.4.16
+ eslint-plugin-unused-imports:
+ specifier: ^3.1.0
+ prettier-plugin-organize-imports:
+ specifier: ^3.2.4
+
packages:
@@ -2048,6 +2085,9 @@ packages:
resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==}
+ resolution: {integrity: sha512-RD7tyk24pNCDwEKFACauR6Lqp5m6BHUrehwyhN/pA8V3QYWq8Y+hk9vHZvKiThZsdEFTaUqN49duVsamgCd8/g==}
+
resolution: {integrity: sha512-aVNobHnJqLiUelTaHat9DZ1qM2w0C0Eym4LPI/3JxOnSokGVdsl1T1kN7TFvsEAD8G47A6VKQ0TVHqbBnYMJlQ==}
engines: {node: '>=12.0.0'}
@@ -5267,6 +5307,8 @@ snapshots:
+
dependencies:
open: 8.4.2
@@ -6543,8 +6585,6 @@ snapshots:
-
dependencies:
tailwindcss: 3.4.17