Skip to content

πŸ™Œ OpenHands: AI-Driven Development

fix(chat): catch up with the model within a second instead of ten (#18651)

Vasco Schiavoa35d32f360d6
diff --git a/.pr/streaming-catch-up/after-fully-shown.png b/.pr/streaming-catch-up/after-fully-shown.png
new file mode 100644
index 0000000000..03d326c3f4
Binary files /dev/null and b/.pr/streaming-catch-up/after-fully-shown.png differ
diff --git a/.pr/streaming-catch-up/before-model-done-plus-10s.png b/.pr/streaming-catch-up/before-model-done-plus-10s.png
new file mode 100644
index 0000000000..b09ee0782f
Binary files /dev/null and b/.pr/streaming-catch-up/before-model-done-plus-10s.png differ
diff --git a/__tests__/hooks/use-streamed-text.test.tsx b/__tests__/hooks/use-streamed-text.test.tsx
index 7f675f0e54..ce712b26bf 100644
--- a/__tests__/hooks/use-streamed-text.test.tsx
+++ b/__tests__/hooks/use-streamed-text.test.tsx
@@ -71,10 +71,11 @@ describe("useStreamedText", () => {
expect(result.current.length).toBeGreaterThan("Hi".length);
expect(result.current.length).toBeLessThan(202);
- // At ~60 chars/s, 200 characters take about 3.3s rather than one burst.
+ // A 200-character burst is spread over about 1.5s rather than shown at
+ // once: quickly at first, then at the reading pace for the last of it.
+ frames.tick(60);
+ expect(result.current.length).toBeLessThan(202);
frames.tick(60);
- expect(result.current.length).toBeLessThan(100);
- frames.tick(160);
expect(result.current).toBe(`Hi${"x".repeat(200)}`);
expect(frames.pending()).toBe(0);
});
@@ -105,7 +106,7 @@ describe("useStreamedText", () => {
expect(Math.max(...halves)).toBeLessThan(1.5);
});
- it("caps the pace for a fast model but never trails it far", () => {
+ it("follows a fast model closely and catches up once it stops", () => {
const { result, rerender } = renderHook(
({ text }) => useStreamedText(text),
{ initialProps: { text: "" } },
@@ -120,13 +121,15 @@ describe("useStreamedText", () => {
}
};
- stream(62); // ~1s
- // Revealed at a reading pace, not at the model's 600 chars/s.
- expect(result.current.length).toBeLessThan(100);
+ stream(300); // ~5s
+ // Smoothed rather than shown as it lands, but never more than about half
+ // a second of the model's output behind.
+ expect(result.current.length).toBeLessThan(length);
+ expect(length - result.current.length).toBeLessThan(600 * 0.6);
- stream(1180); // ~20s in total
- // Past ~10s of backlog it speeds up, so it never trails further.
- expect(length - result.current.length).toBeLessThan(600 * 10);
+ frames.tick(120); // ~2s after the model stopped
+ expect(result.current).toBe("x".repeat(length));
+ expect(frames.pending()).toBe(0);
});
it("reports while text is still being revealed", () => {
diff --git a/src/hooks/use-streamed-text.ts b/src/hooks/use-streamed-text.ts
index 74e5483851..327f966bc7 100644
--- a/src/hooks/use-streamed-text.ts
+++ b/src/hooks/use-streamed-text.ts
@@ -1,19 +1,15 @@
import React from "react";
import { create } from "zustand";
-/**
- * Reveal pace, in characters per millisecond (60 chars/s). Fast models deliver
- * several hundred characters a second; revealing at that pace reads as text
- * being dumped rather than written.
- */
+/** Reveal pace, in characters per millisecond (60 chars/s). */
const PACE = 0.06;
/**
- * How far the reveal may trail the model before it speeds up beyond `PACE`:
- * past this backlog, it drains at least `backlog / MAX_LAG_MS`, so a long
- * reply from a fast model never ends up more than ~10 seconds behind.
+ * Backlog is drained at `backlog / MAX_LAG_MS` per ms once that exceeds
+ * `PACE`, so the reveal trails a fast model by about this long and catches
+ * up within about a second of it stopping.
*/
-const MAX_LAG_MS = 10000;
+const MAX_LAG_MS = 500;
/**
* Hooks still revealing text, so the chat can keep following a reply that is