The Open Standard for Generative UI
Stream Improvements (#80)
Rabi Shanker Guhac6d78fab954f
diff --git a/js/packages/stream/package.json b/js/packages/stream/package.jsonindex 195e6667f..5c5f038a0 100644--- a/js/packages/stream/package.json+++ b/js/packages/stream/package.json@@ -10,6 +10,7 @@"scripts": {"test": "echo \"Error: no test specified\" && exit 1","build": "tsc -p .",+ "watch": "tsc -p . --watch","lint:check": "eslint ./src","lint:fix": "eslint ./src --fix","format:fix": "prettier --write ./src",diff --git a/js/packages/stream/src/anthropic.ts b/js/packages/stream/src/anthropic.tsnew file mode 100644index 000000000..1a5e8732e--- /dev/null+++ b/js/packages/stream/src/anthropic.ts@@ -0,0 +1,29 @@+// These types are defined here so as to not introduce a dependency+// on anthropic-ai-sdk library directly.+interface AnthropicAIMessage {+ role: "user" | "assistant";+ content: string;+}++export const toAnthropicAIMessages = (messages: any[]) => {+ const anthropicAIMessages: AnthropicAIMessage[] = [];+ for (const message of messages) {+ if (!message.message) {+ continue;+ }+ if (message.role === "user") {+ anthropicAIMessages.push({+ role: message.role,+ content: message.message,+ });+ } else if (message.role === "assistant") {+ anthropicAIMessages.push({+ role: message.role,+ content: JSON.stringify({+ response: message.message,+ }),+ });+ }+ }+ return anthropicAIMessages;+};diff --git a/js/packages/stream/src/crayonStream.ts b/js/packages/stream/src/crayonStream.tsnew file mode 100644index 000000000..ccca2bead--- /dev/null+++ b/js/packages/stream/src/crayonStream.ts@@ -0,0 +1,32 @@+import { ReadableStreamController } from "stream/web";+import { CrayonDataStreamTransformer, TransformerOpts } from "./transformer";++export const crayonStream = (opts?: TransformerOpts) => {+ let controller: ReadableStreamController<string> | null = null;+ const readableStream = new ReadableStream({+ async start(c) {+ controller = c;+ },+ });+ const crayonTransformer = new CrayonDataStreamTransformer(opts);+ const onText = (text: string) => {+ controller!.enqueue(text);+ };+ const onEnd = () => {+ controller!.close();+ };+ const onError = (e: Error) => {+ controller!.error(e);+ };+ const onLLMEnd = () => {+ crayonTransformer.reset();+ };++ return {+ stream: readableStream.pipeThrough(crayonTransformer),+ onText,+ onEnd,+ onLLMEnd,+ onError,+ };+};diff --git a/js/packages/stream/src/index.ts b/js/packages/stream/src/index.tsindex bd9b99ede..c786e239e 100644--- a/js/packages/stream/src/index.ts+++ b/js/packages/stream/src/index.ts@@ -1,3 +1,5 @@+export * from "./anthropic";+export * from "./crayonStream";export * from "./jsonSchemaHelpers";export * from "./openai";export * from "./textSchema";diff --git a/js/packages/stream/src/openai.ts b/js/packages/stream/src/openai.tsindex b4627225b..a1de744a4 100644--- a/js/packages/stream/src/openai.ts+++ b/js/packages/stream/src/openai.ts@@ -1,4 +1,5 @@-import { CrayonDataStreamTransformer, TransformerOpts } from "./transformer";+import { crayonStream } from "./crayonStream";+import { TransformerOpts } from "./transformer";// These types are defined here so as to not introduce a dependency// on openai library directly.@@ -20,22 +21,24 @@ export const fromOpenAICompletion = (completion: ChatCompletionStreamingRunner,opts?: TransformerOpts,) => {- const readableStream = new ReadableStream({- async start(controller) {- try {- for await (const chunk of completion) {- const content = chunk.choices[0]?.delta?.content;- if (content) {- controller.enqueue(content);- }- }- controller.close();- } catch (error) {- controller.error(error);+ const { stream, onText, onEnd, onError, onLLMEnd } = crayonStream(opts);+ (async () => {+ try {+ for await (const chunk of completion) {+ const content = chunk.choices[0]?.delta?.content;+ content && onText(content);}- },- });- return readableStream.pipeThrough(new CrayonDataStreamTransformer(opts));+ onLLMEnd();+ onEnd();+ } catch (error) {+ if (error instanceof Error) {+ onError(error);+ } else {+ onError(new Error(String(error)));+ }+ }+ })();+ return stream;};// This is a utility function to convert a Crayon message to an OpenAI message.@@ -49,10 +52,19 @@ export const toOpenAIMessages = (messages: any[]) => {if (!message.message) {continue;}- openAIMessages.push({- role: message.role,- content: JSON.stringify(message.message),- });+ if (message.role === "user") {+ openAIMessages.push({+ role: message.role,+ content: message.message,+ });+ } else if (message.role === "assistant") {+ openAIMessages.push({+ role: message.role,+ content: JSON.stringify({+ response: message.message,+ }),+ });+ }}return openAIMessages;};diff --git a/js/packages/stream/src/textSchema.ts b/js/packages/stream/src/textSchema.tsindex 140eaeda1..9621f0851 100644--- a/js/packages/stream/src/textSchema.ts+++ b/js/packages/stream/src/textSchema.ts@@ -5,9 +5,10 @@ export const TextResponseSchema = {type: "object",properties: {type: { const: "text" },+ description: "Use this as a fallback when no other response template is applicable",text: {type: "string",- description: "text message to be displayed to the user",+ description: "plaintext message to be displayed to the user",},},required: ["type", "text"],diff --git a/js/packages/stream/src/transformer.ts b/js/packages/stream/src/transformer.tsindex 594877366..02f116814 100644--- a/js/packages/stream/src/transformer.ts+++ b/js/packages/stream/src/transformer.ts@@ -12,32 +12,49 @@ export interface TransformerOpts {export class CrayonDataStreamTransformer implements TransformStream<InputType, OutputType> {readonly readable: ReadableStream<OutputType>;readonly writable: WritableStream<InputType>;+ private streamedContent: string = "";+ private controller: TransformStreamDefaultController<OutputType> | null = null;+ private hasPendingTemplateToStream: boolean = false;++ reset() {+ const parsed = parse(this.streamedContent);+ const lastContent = parsed?.response?.pop?.();++ if (this.hasPendingTemplateToStream && lastContent && lastContent.type !== "text") {+ const { name, templateProps } = lastContent;+ invariant(name, "name is required in ResponseTemplate");+ this.controller?.enqueue(new ResponseTemplate(name, templateProps).toSSEString());+ }+ this.hasPendingTemplateToStream = false;+ this.streamedContent = "";+ }constructor(opts?: TransformerOpts) {- let streamedContent = "";+ this.hasPendingTemplateToStream = false;+ this.streamedContent = "";let previouslyStreamedTextContent: string = "";- let hasPendingTemplateToStream: boolean = false;const transform = new TransformStream({transform: async (content: InputType,controller: TransformStreamDefaultController<OutputType>,) => {+ this.controller = controller;try {- streamedContent += content;+ this.streamedContent += content;- const parsed = parse(streamedContent);+ const parsed = parse(this.streamedContent);const newContent = parsed?.response?.pop?.();if (newContent?.type === "text") {const prevContent = parsed.response.pop();const newText = newContent.text || "";- if (hasPendingTemplateToStream && prevContent && prevContent?.type !== "text") {+ if (this.hasPendingTemplateToStream && prevContent && prevContent?.type !== "text") {const { name, templateProps } = prevContent;invariant(name, "name is required in ResponseTemplate");controller.enqueue(new ResponseTemplate(name, templateProps).toSSEString());- hasPendingTemplateToStream = false;+ this.hasPendingTemplateToStream = false;}const textContent = newText.substring(previouslyStreamedTextContent.length);@@ -48,7 +65,7 @@ export class CrayonDataStreamTransformer implements TransformStream<InputType, OpreviouslyStreamedTextContent = newText;} else {- hasPendingTemplateToStream = true;+ this.hasPendingTemplateToStream = true;previouslyStreamedTextContent = "";}} catch (error) {@@ -57,16 +74,9 @@ export class CrayonDataStreamTransformer implements TransformStream<InputType, O},flush: async (controller: TransformStreamDefaultController<OutputType>) => {- const parsed = parse(streamedContent);- const lastContent = parsed?.response?.pop?.();-- if (hasPendingTemplateToStream && lastContent && lastContent.type !== "text") {- const { name, templateProps } = lastContent;- invariant(name, "name is required in ResponseTemplate");- controller.enqueue(new ResponseTemplate(name, templateProps).toSSEString());- }-+ this.reset();await opts?.onFinish(controller);+ this.controller = null;},});