Skip to content

armgit / E2B

Public Imported from GitHub
0

Open-source, secure environment with real-world tools for enterprise-grade agents.

fix(code-interpreter): preserve UTF-8 across response chunks (#1946)

devin-ai-integration[bot]af8d7a5885f3
diff --git a/.changeset/calm-streamed-unicode.md b/.changeset/calm-streamed-unicode.md
new file mode 100644
index 000000000..4cf4ee09f
--- /dev/null
+++ b/.changeset/calm-streamed-unicode.md
@@ -0,0 +1,5 @@
+---
+'@e2b/code-interpreter': patch
+---
+
+Preserve Unicode characters split across response chunks in code execution output.
diff --git a/packages/code-interpreter-js/src/utils.ts b/packages/code-interpreter-js/src/utils.ts
index 1bb8ff29b..64a2e1f6b 100644
--- a/packages/code-interpreter-js/src/utils.ts
+++ b/packages/code-interpreter-js/src/utils.ts
@@ -65,6 +65,7 @@ export function isConnectionClosedError(error: unknown): boolean {
export async function* readLines(stream: ReadableStream<Uint8Array>) {
const reader = stream.getReader()
+ const decoder = new TextDecoder()
let buffer = ''
try {
@@ -72,10 +73,11 @@ export async function* readLines(stream: ReadableStream<Uint8Array>) {
const { done, value } = await reader.read()
if (value !== undefined) {
- buffer += new TextDecoder().decode(value)
+ buffer += decoder.decode(value, { stream: true })
}
if (done) {
+ buffer += decoder.decode()
if (buffer.length > 0) {
yield buffer
}
diff --git a/packages/code-interpreter-js/tests/utils.test.ts b/packages/code-interpreter-js/tests/utils.test.ts
new file mode 100644
index 000000000..6efc7f03a
--- /dev/null
+++ b/packages/code-interpreter-js/tests/utils.test.ts
@@ -0,0 +1,64 @@
+import { expect, test } from 'vitest'
+
+import { readLines } from '../src/utils'
+
+test.each([1, 2, 3, 4, 7])(
+ 'preserves UTF-8 output split into %s-byte chunks',
+ async (chunkSize) => {
+ const expected = ['{"text":"你好 🌍 café"}', '{"text":"日本語 🚀"}']
+ const bytes = new TextEncoder().encode(expected.join('\n'))
+ const stream = new ReadableStream<Uint8Array>({
+ start(controller) {
+ for (let offset = 0; offset < bytes.length; offset += chunkSize) {
+ controller.enqueue(bytes.slice(offset, offset + chunkSize))
+ }
+ controller.close()
+ },
+ })
+ const lines: string[] = []
+
+ for await (const line of readLines(stream)) {
+ lines.push(line)
+ }
+
+ expect(lines).toEqual(expected)
+ expect(stream.locked).toBe(false)
+ }
+)
+
+test.each(['', 'first\n\nlast\n', 'first\nlast', '🌍\n你好\n'])(
+ 'preserves complete lines and an unterminated final line: %j',
+ async (text) => {
+ const stream = new ReadableStream<Uint8Array>({
+ start(controller) {
+ controller.enqueue(new TextEncoder().encode(text))
+ controller.close()
+ },
+ })
+ const lines: string[] = []
+
+ for await (const line of readLines(stream)) {
+ lines.push(line)
+ }
+
+ const expected = text.split('\n')
+ if (expected.at(-1) === '') expected.pop()
+ expect(lines).toEqual(expected)
+ }
+)
+
+test('flushes an incomplete UTF-8 sequence when the stream ends', async () => {
+ const stream = new ReadableStream<Uint8Array>({
+ start(controller) {
+ controller.enqueue(new Uint8Array([0xe4, 0xbd]))
+ controller.close()
+ },
+ })
+ const lines: string[] = []
+
+ for await (const line of readLines(stream)) {
+ lines.push(line)
+ }
+
+ expect(lines).toEqual(['\uFFFD'])
+})