A platform for community discussion. Free, open, simple.
PERF: Speed up AI helper diff streaming (#41787)
Rafael dos Santos Silva53e25da301ae
diff --git a/plugins/discourse-ai/assets/javascripts/discourse/lib/diff-streamer.gjs b/plugins/discourse-ai/assets/javascripts/discourse/lib/diff-streamer.gjsindex 49fbb146bb7..c29a60b82bc 100644--- a/plugins/discourse-ai/assets/javascripts/discourse/lib/diff-streamer.gjs+++ b/plugins/discourse-ai/assets/javascripts/discourse/lib/diff-streamer.gjs@@ -8,6 +8,8 @@ const STREAMING_DIFF_TRUNCATE_THRESHOLD = 0.1;const STREAMING_DIFF_TRUNCATE_BUFFER = 10;const RUSH_MAX_TICKS = 10; // ≤ 10 visual diff refreshesconst RUSH_TICK_INTERVAL = 100; // 100 ms between them → ≤ 1 s total+const BACKLOG_DRAIN_TARGET = 500; // aim to catch up with received text within 500 ms+const DIFF_REFRESH_INTERVAL = 100; // minimum ms between streaming diff recomputesexport default class DiffStreamer {@tracked isStreaming = false;@@ -21,6 +23,7 @@ export default class DiffStreamer {typingTimer = null;currentWordIndex = 0;currentCharIndex = 0;+ lastDiffRefreshedAt = 0;jsDiff = null;bufferedToken = null;@@ -142,6 +145,7 @@ export default class DiffStreamer {this.words = [];this.currentWordIndex = 0;this.currentCharIndex = 0;+ this.lastDiffRefreshedAt = 0;this.bufferedToken = null;this.isStreaming = false;@@ -203,9 +207,16 @@ export default class DiffStreamer {return;}- const limit = this.rushMode ? this.rushBatchSize : 1;+ const limit = this.rushMode+ ? this.rushBatchSize+ : this.#adaptiveBatchSize();let emitted = 0;- while (emitted < limit && this.currentWordIndex < this.words.length) {+ // once the limit is reached, keep going to the end of the current word so+ // diff refreshes always happen on word boundaries+ while (+ this.currentWordIndex < this.words.length &&+ (emitted < limit || this.currentCharIndex !== 0)+ ) {const token = this.words[this.currentWordIndex];this.suggestion += token.charAt(this.currentCharIndex);this.currentCharIndex++;@@ -224,7 +235,7 @@ export default class DiffStreamer {refresh = true;}} else {- refresh = this.currentCharIndex === 0;+ refresh = Date.now() - this.lastDiffRefreshedAt >= DIFF_REFRESH_INTERVAL;}if (refresh || this.currentWordIndex >= this.words.length) {@@ -236,6 +247,7 @@ export default class DiffStreamer {: this.jsDiff.diffWordsWithSpace(this.selectedText, this.suggestion),!this.rushMode);+ this.lastDiffRefreshedAt = Date.now();}const doneStreaming = this.currentWordIndex >= this.words.length;@@ -254,6 +266,18 @@ export default class DiffStreamer {}}+ /**+ * Computes how many characters to emit this tick so the animation drains+ * its backlog within BACKLOG_DRAIN_TARGET instead of being capped at one+ * character per tick regardless of how fast tokens actually arrive.+ * @returns {number} Characters to emit on the next tick.+ */+ #adaptiveBatchSize() {+ const backlog = this.lastResultText.length - this.suggestion.length;+ const ticksInTarget = BACKLOG_DRAIN_TARGET / this.typingDelay;+ return Math.max(1, Math.ceil(backlog / ticksInTarget));+ }+/*** Splits a string into tokens, preserving whitespace as separate entries.* @param {string} text - The input string.diff --git a/plugins/discourse-ai/lib/ai_helper/assistant.rb b/plugins/discourse-ai/lib/ai_helper/assistant.rbindex dbd1df8234b..0fd3394a8ff 100644--- a/plugins/discourse-ai/lib/ai_helper/assistant.rb+++ b/plugins/discourse-ai/lib/ai_helper/assistant.rb@@ -216,7 +216,6 @@ module DiscourseAiclient_id: nil,custom_prompt: nil)- streamed_diff = +""streamed_result = +""start = Time.nowtype = prompt_type(helper_mode)@@ -229,13 +228,12 @@ module DiscourseAicustom_prompt: custom_prompt,) do |partial_response|streamed_result << partial_response- streamed_diff = parse_diff(input, partial_response) if type == :diff- # Throttle updates and check for safe stream points+ # Throttle updatesif (streamed_result.length > 10 && (Time.now - start > 0.3)) || Rails.env.test?sanitized = sanitize_result(streamed_result)- payload = { result: sanitized, diff: streamed_diff, done: false }+ payload = { result: sanitized, done: false }publish_update(channel, payload, user, client_id: client_id)start = Time.nowend