Open-source, secure environment with real-world tools for enterprise-grade agents.
Python agentaiai-agentai-agentscode-interpretercopilotdevelopmentdevtoolsgptgpt-4javascriptllmnextjsopenaipythonreactsoftwaretypescript
fix(code-interpreter): preserve UTF-8 across response chunks (#1946)
devin-ai-integration[bot]af8d7a5885f3
diff --git a/.changeset/calm-streamed-unicode.md b/.changeset/calm-streamed-unicode.mdnew file mode 100644index 000000000..4cf4ee09f--- /dev/null+++ b/.changeset/calm-streamed-unicode.md@@ -0,0 +1,5 @@+---+'@e2b/code-interpreter': patch+---++Preserve Unicode characters split across response chunks in code execution output.diff --git a/packages/code-interpreter-js/src/utils.ts b/packages/code-interpreter-js/src/utils.tsindex 1bb8ff29b..64a2e1f6b 100644--- a/packages/code-interpreter-js/src/utils.ts+++ b/packages/code-interpreter-js/src/utils.ts@@ -65,6 +65,7 @@ export function isConnectionClosedError(error: unknown): boolean {export async function* readLines(stream: ReadableStream<Uint8Array>) {const reader = stream.getReader()+ const decoder = new TextDecoder()let buffer = ''try {@@ -72,10 +73,11 @@ export async function* readLines(stream: ReadableStream<Uint8Array>) {const { done, value } = await reader.read()if (value !== undefined) {- buffer += new TextDecoder().decode(value)+ buffer += decoder.decode(value, { stream: true })}if (done) {+ buffer += decoder.decode()if (buffer.length > 0) {yield buffer}diff --git a/packages/code-interpreter-js/tests/utils.test.ts b/packages/code-interpreter-js/tests/utils.test.tsnew file mode 100644index 000000000..6efc7f03a--- /dev/null+++ b/packages/code-interpreter-js/tests/utils.test.ts@@ -0,0 +1,64 @@+import { expect, test } from 'vitest'++import { readLines } from '../src/utils'++test.each([1, 2, 3, 4, 7])(+ 'preserves UTF-8 output split into %s-byte chunks',+ async (chunkSize) => {+ const expected = ['{"text":"你好 🌍 café"}', '{"text":"日本語 🚀"}']+ const bytes = new TextEncoder().encode(expected.join('\n'))+ const stream = new ReadableStream<Uint8Array>({+ start(controller) {+ for (let offset = 0; offset < bytes.length; offset += chunkSize) {+ controller.enqueue(bytes.slice(offset, offset + chunkSize))+ }+ controller.close()+ },+ })+ const lines: string[] = []++ for await (const line of readLines(stream)) {+ lines.push(line)+ }++ expect(lines).toEqual(expected)+ expect(stream.locked).toBe(false)+ }+)++test.each(['', 'first\n\nlast\n', 'first\nlast', '🌍\n你好\n'])(+ 'preserves complete lines and an unterminated final line: %j',+ async (text) => {+ const stream = new ReadableStream<Uint8Array>({+ start(controller) {+ controller.enqueue(new TextEncoder().encode(text))+ controller.close()+ },+ })+ const lines: string[] = []++ for await (const line of readLines(stream)) {+ lines.push(line)+ }++ const expected = text.split('\n')+ if (expected.at(-1) === '') expected.pop()+ expect(lines).toEqual(expected)+ }+)++test('flushes an incomplete UTF-8 sequence when the stream ends', async () => {+ const stream = new ReadableStream<Uint8Array>({+ start(controller) {+ controller.enqueue(new Uint8Array([0xe4, 0xbd]))+ controller.close()+ },+ })+ const lines: string[] = []++ for await (const line of readLines(stream)) {+ lines.push(line)+ }++ expect(lines).toEqual(['\uFFFD'])+})