Skip to content

Distribute and run LLMs with a single file.

Visually indicate messages truncated by context

Justine Tunney956e62ce2765
diff --git a/llamafile/server/www/chatbot.css b/llamafile/server/www/chatbot.css
index 7b1e83a1..7d74b2d1 100644
--- a/llamafile/server/www/chatbot.css
+++ b/llamafile/server/www/chatbot.css
@@ -675,3 +675,8 @@ ul li:first-child {
width: 0;
transition: width 0.2s ease-out;
}
+
+.ooc {
+ vertical-align: middle;
+ margin-left: 0.2rem;
+}
diff --git a/llamafile/server/www/chatbot.js b/llamafile/server/www/chatbot.js
index bb4a8b11..ad68669b 100644
--- a/llamafile/server/www/chatbot.js
+++ b/llamafile/server/www/chatbot.js
@@ -132,6 +132,7 @@ async function handleChatStream(response) {
try {
const parsed = JSON.parse(data);
const content = parsed.choices[0]?.delta?.content || "";
+ const finishReason = parsed.choices[0]?.finish_reason;
// handle prefill progress
if (parsed.x_prefill_progress !== undefined) {
@@ -154,6 +155,18 @@ async function handleChatStream(response) {
high.feed(content);
scrollToBottom();
}
+
+ // we don't supply max_tokens, so "length" can
+ // only mean that we ran out of context window
+ if (finishReason === "length" && hdom) {
+ let img = hdom.push("IMG", "ooc");
+ img.src = "ooc.svg";
+ img.alt = "🚫";
+ img.title = "Message truncated due to running out of context window. Consider tuning --ctx-size and/or --reserve-tokens";
+ img.width = 16;
+ img.height = 16;
+ hdom.pop();
+ }
} catch (e) {
console.error("Error parsing JSON:", e);
}
diff --git a/llamafile/server/www/ooc.svg b/llamafile/server/www/ooc.svg
new file mode 100644
index 00000000..c8859c72
--- /dev/null
+++ b/llamafile/server/www/ooc.svg
@@ -0,0 +1,15 @@
+<!-- out of context-->
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
+
+ <!-- outer circle -->
+ <circle cx="50" cy="50" r="45"
+ fill="none"
+ stroke="red"
+ stroke-width="8"/>
+
+ <!-- diagonal line -->
+ <line x1="15" y1="85"
+ x2="85" y2="15"
+ stroke="red"
+ stroke-width="8"/>
+</svg>