Skip to content

The Open Standard for Generative UI

Stream Improvements (#80)

Rabi Shanker Guhac6d78fab954f
diff --git a/js/packages/stream/package.json b/js/packages/stream/package.json
index 195e6667f..5c5f038a0 100644
--- a/js/packages/stream/package.json
+++ b/js/packages/stream/package.json
@@ -10,6 +10,7 @@
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"build": "tsc -p .",
+ "watch": "tsc -p . --watch",
"lint:check": "eslint ./src",
"lint:fix": "eslint ./src --fix",
"format:fix": "prettier --write ./src",
diff --git a/js/packages/stream/src/anthropic.ts b/js/packages/stream/src/anthropic.ts
new file mode 100644
index 000000000..1a5e8732e
--- /dev/null
+++ b/js/packages/stream/src/anthropic.ts
@@ -0,0 +1,29 @@
+// These types are defined here so as to not introduce a dependency
+// on anthropic-ai-sdk library directly.
+interface AnthropicAIMessage {
+ role: "user" | "assistant";
+ content: string;
+}
+
+export const toAnthropicAIMessages = (messages: any[]) => {
+ const anthropicAIMessages: AnthropicAIMessage[] = [];
+ for (const message of messages) {
+ if (!message.message) {
+ continue;
+ }
+ if (message.role === "user") {
+ anthropicAIMessages.push({
+ role: message.role,
+ content: message.message,
+ });
+ } else if (message.role === "assistant") {
+ anthropicAIMessages.push({
+ role: message.role,
+ content: JSON.stringify({
+ response: message.message,
+ }),
+ });
+ }
+ }
+ return anthropicAIMessages;
+};
diff --git a/js/packages/stream/src/crayonStream.ts b/js/packages/stream/src/crayonStream.ts
new file mode 100644
index 000000000..ccca2bead
--- /dev/null
+++ b/js/packages/stream/src/crayonStream.ts
@@ -0,0 +1,32 @@
+import { ReadableStreamController } from "stream/web";
+import { CrayonDataStreamTransformer, TransformerOpts } from "./transformer";
+
+export const crayonStream = (opts?: TransformerOpts) => {
+ let controller: ReadableStreamController<string> | null = null;
+ const readableStream = new ReadableStream({
+ async start(c) {
+ controller = c;
+ },
+ });
+ const crayonTransformer = new CrayonDataStreamTransformer(opts);
+ const onText = (text: string) => {
+ controller!.enqueue(text);
+ };
+ const onEnd = () => {
+ controller!.close();
+ };
+ const onError = (e: Error) => {
+ controller!.error(e);
+ };
+ const onLLMEnd = () => {
+ crayonTransformer.reset();
+ };
+
+ return {
+ stream: readableStream.pipeThrough(crayonTransformer),
+ onText,
+ onEnd,
+ onLLMEnd,
+ onError,
+ };
+};
diff --git a/js/packages/stream/src/index.ts b/js/packages/stream/src/index.ts
index bd9b99ede..c786e239e 100644
--- a/js/packages/stream/src/index.ts
+++ b/js/packages/stream/src/index.ts
@@ -1,3 +1,5 @@
+export * from "./anthropic";
+export * from "./crayonStream";
export * from "./jsonSchemaHelpers";
export * from "./openai";
export * from "./textSchema";
diff --git a/js/packages/stream/src/openai.ts b/js/packages/stream/src/openai.ts
index b4627225b..a1de744a4 100644
--- a/js/packages/stream/src/openai.ts
+++ b/js/packages/stream/src/openai.ts
@@ -1,4 +1,5 @@
-import { CrayonDataStreamTransformer, TransformerOpts } from "./transformer";
+import { crayonStream } from "./crayonStream";
+import { TransformerOpts } from "./transformer";
// These types are defined here so as to not introduce a dependency
// on openai library directly.
@@ -20,22 +21,24 @@ export const fromOpenAICompletion = (
completion: ChatCompletionStreamingRunner,
opts?: TransformerOpts,
) => {
- const readableStream = new ReadableStream({
- async start(controller) {
- try {
- for await (const chunk of completion) {
- const content = chunk.choices[0]?.delta?.content;
- if (content) {
- controller.enqueue(content);
- }
- }
- controller.close();
- } catch (error) {
- controller.error(error);
+ const { stream, onText, onEnd, onError, onLLMEnd } = crayonStream(opts);
+ (async () => {
+ try {
+ for await (const chunk of completion) {
+ const content = chunk.choices[0]?.delta?.content;
+ content && onText(content);
}
- },
- });
- return readableStream.pipeThrough(new CrayonDataStreamTransformer(opts));
+ onLLMEnd();
+ onEnd();
+ } catch (error) {
+ if (error instanceof Error) {
+ onError(error);
+ } else {
+ onError(new Error(String(error)));
+ }
+ }
+ })();
+ return stream;
};
// This is a utility function to convert a Crayon message to an OpenAI message.
@@ -49,10 +52,19 @@ export const toOpenAIMessages = (messages: any[]) => {
if (!message.message) {
continue;
}
- openAIMessages.push({
- role: message.role,
- content: JSON.stringify(message.message),
- });
+ if (message.role === "user") {
+ openAIMessages.push({
+ role: message.role,
+ content: message.message,
+ });
+ } else if (message.role === "assistant") {
+ openAIMessages.push({
+ role: message.role,
+ content: JSON.stringify({
+ response: message.message,
+ }),
+ });
+ }
}
return openAIMessages;
};
diff --git a/js/packages/stream/src/textSchema.ts b/js/packages/stream/src/textSchema.ts
index 140eaeda1..9621f0851 100644
--- a/js/packages/stream/src/textSchema.ts
+++ b/js/packages/stream/src/textSchema.ts
@@ -5,9 +5,10 @@ export const TextResponseSchema = {
type: "object",
properties: {
type: { const: "text" },
+ description: "Use this as a fallback when no other response template is applicable",
text: {
type: "string",
- description: "text message to be displayed to the user",
+ description: "plaintext message to be displayed to the user",
},
},
required: ["type", "text"],
diff --git a/js/packages/stream/src/transformer.ts b/js/packages/stream/src/transformer.ts
index 594877366..02f116814 100644
--- a/js/packages/stream/src/transformer.ts
+++ b/js/packages/stream/src/transformer.ts
@@ -12,32 +12,49 @@ export interface TransformerOpts {
export class CrayonDataStreamTransformer implements TransformStream<InputType, OutputType> {
readonly readable: ReadableStream<OutputType>;
readonly writable: WritableStream<InputType>;
+ private streamedContent: string = "";
+ private controller: TransformStreamDefaultController<OutputType> | null = null;
+ private hasPendingTemplateToStream: boolean = false;
+
+ reset() {
+ const parsed = parse(this.streamedContent);
+ const lastContent = parsed?.response?.pop?.();
+
+ if (this.hasPendingTemplateToStream && lastContent && lastContent.type !== "text") {
+ const { name, templateProps } = lastContent;
+ invariant(name, "name is required in ResponseTemplate");
+ this.controller?.enqueue(new ResponseTemplate(name, templateProps).toSSEString());
+ }
+ this.hasPendingTemplateToStream = false;
+ this.streamedContent = "";
+ }
constructor(opts?: TransformerOpts) {
- let streamedContent = "";
+ this.hasPendingTemplateToStream = false;
+ this.streamedContent = "";
let previouslyStreamedTextContent: string = "";
- let hasPendingTemplateToStream: boolean = false;
const transform = new TransformStream({
transform: async (
content: InputType,
controller: TransformStreamDefaultController<OutputType>,
) => {
+ this.controller = controller;
try {
- streamedContent += content;
+ this.streamedContent += content;
- const parsed = parse(streamedContent);
+ const parsed = parse(this.streamedContent);
const newContent = parsed?.response?.pop?.();
if (newContent?.type === "text") {
const prevContent = parsed.response.pop();
const newText = newContent.text || "";
- if (hasPendingTemplateToStream && prevContent && prevContent?.type !== "text") {
+ if (this.hasPendingTemplateToStream && prevContent && prevContent?.type !== "text") {
const { name, templateProps } = prevContent;
invariant(name, "name is required in ResponseTemplate");
controller.enqueue(new ResponseTemplate(name, templateProps).toSSEString());
- hasPendingTemplateToStream = false;
+ this.hasPendingTemplateToStream = false;
}
const textContent = newText.substring(previouslyStreamedTextContent.length);
@@ -48,7 +65,7 @@ export class CrayonDataStreamTransformer implements TransformStream<InputType, O
previouslyStreamedTextContent = newText;
} else {
- hasPendingTemplateToStream = true;
+ this.hasPendingTemplateToStream = true;
previouslyStreamedTextContent = "";
}
} catch (error) {
@@ -57,16 +74,9 @@ export class CrayonDataStreamTransformer implements TransformStream<InputType, O
},
flush: async (controller: TransformStreamDefaultController<OutputType>) => {
- const parsed = parse(streamedContent);
- const lastContent = parsed?.response?.pop?.();
-
- if (hasPendingTemplateToStream && lastContent && lastContent.type !== "text") {
- const { name, templateProps } = lastContent;
- invariant(name, "name is required in ResponseTemplate");
- controller.enqueue(new ResponseTemplate(name, templateProps).toSSEString());
- }
-
+ this.reset();
await opts?.onFinish(controller);
+ this.controller = null;
},
});