Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Prune chat history in token quanta

Manav Rathib53f2eb63789
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
index a100c14990..f57c0fcb1b 100644
--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
@@ -1099,31 +1099,37 @@ internal class ChatStoreActions(
val promptTokens = estimatePromptTokens(promptText, promptImageCount)
val historyTokens = historyMessages.sumOf { estimateTokens(historyText(it)) }
val inputTokens = systemTokens + promptTokens + historyTokens
- var remaining = inputBudget - systemTokens - promptTokens
+ val remaining = inputBudget - systemTokens - promptTokens
if (remaining <= 0 || historyMessages.isEmpty()) {
return HistorySelection(emptyList(), inputTokens, inputBudget, inputTokens > inputBudget)
}
- val selected = mutableListOf<LlmMessage>()
- for (message in historyMessages.asReversed()) {
+ val quantum = max(1, inputBudget / 4)
+ val overflow = max(0, historyTokens - remaining)
+ val quantaToDiscard = (overflow + quantum - 1) / quantum
+ val discardTarget = quantaToDiscard * quantum
+ var discarded = 0
+ var startIndex = 0
+ while (startIndex < historyMessages.size && discarded < discardTarget) {
+ discarded += estimateTokens(historyText(historyMessages[startIndex]))
+ startIndex++
+ }
+
+ val retained = historyMessages.drop(startIndex).ifEmpty {
+ historyMessages.takeLast(1).filter { estimateTokens(historyText(it)) <= remaining }
+ }
+
+ val selected = retained.map { message ->
val text = historyText(message)
- val cost = estimateTokens(text)
- if (cost <= remaining) {
- selected.add(
- LlmMessage(
- text = text,
- role = if (message.author == MessageAuthor.User) LlmMessageRole.User else LlmMessageRole.Assistant,
- hasAttachments = message.attachments.isNotEmpty()
- )
- )
- remaining -= cost
- } else {
- break
- }
+ LlmMessage(
+ text = text,
+ role = if (message.author == MessageAuthor.User) LlmMessageRole.User else LlmMessageRole.Assistant,
+ hasAttachments = message.attachments.isNotEmpty()
+ )
}
- return HistorySelection(selected.reversed(), inputTokens, inputBudget, inputTokens > inputBudget)
+ return HistorySelection(selected, inputTokens, inputBudget, inputTokens > inputBudget)
}
private fun resolveGenerationLimits(target: LlmModelTarget): GenerationLimits {
diff --git a/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
index ebbcbc7ae5..1a7eb57a86 100644
--- a/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
+++ b/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
@@ -1847,25 +1847,34 @@ final class ChatViewModel: ObservableObject {
total + estimateTokens(historyText(node))
}
let inputTokens = systemTokens + promptTokens + historyTokens
- var remaining = inputBudget - systemTokens - promptTokens
+ let remaining = inputBudget - systemTokens - promptTokens
if remaining <= 0 || historyMessages.isEmpty {
return HistorySelection(messages: [], inputTokens: inputTokens, inputBudget: inputBudget, wasTrimmed: inputTokens > inputBudget)
}
- var selected: [LlmMessage] = []
- for node in historyMessages.reversed() {
+ let quantum = max(1, inputBudget / 4)
+ let overflow = max(0, historyTokens - remaining)
+ let quantaToDiscard = (overflow + quantum - 1) / quantum
+ let discardTarget = quantaToDiscard * quantum
+ var discarded = 0
+ var startIndex = historyMessages.startIndex
+ while startIndex < historyMessages.endIndex && discarded < discardTarget {
+ discarded += estimateTokens(historyText(historyMessages[startIndex]))
+ startIndex = historyMessages.index(after: startIndex)
+ }
+
+ var retained = historyMessages[startIndex...]
+ if retained.isEmpty, let last = historyMessages.last, estimateTokens(historyText(last)) <= remaining {
+ retained = historyMessages.suffix(1)
+ }
+
+ let selected = retained.map { node in
let text = historyText(node)
- let cost = estimateTokens(text)
- if cost <= remaining {
- selected.append(LlmMessage(text: text, role: node.role == .user ? .user : .assistant, hasAttachments: !node.attachments.isEmpty))
- remaining -= cost
- } else {
- break
- }
+ return LlmMessage(text: text, role: node.role == .user ? .user : .assistant, hasAttachments: !node.attachments.isEmpty)
}
- return HistorySelection(messages: selected.reversed(), inputTokens: inputTokens, inputBudget: inputBudget, wasTrimmed: inputTokens > inputBudget)
+ return HistorySelection(messages: selected, inputTokens: inputTokens, inputBudget: inputBudget, wasTrimmed: inputTokens > inputBudget)
}
private func resolveGenerationLimits(target: LlmModelTarget) -> GenerationLimits {
diff --git a/web/apps/ensu/src/pages/chat.tsx b/web/apps/ensu/src/pages/chat.tsx
index db237fa9a8..c7e294a3a1 100644
--- a/web/apps/ensu/src/pages/chat.tsx
+++ b/web/apps/ensu/src/pages/chat.tsx
@@ -2179,22 +2179,18 @@ const Page: React.FC = () => {
? candidates.slice(0, -1)
: candidates;
- const safetyMargin = 256;
- let budget =
+ const inputBudget =
contextSize -
(maxTokensCount ?? DEFAULT_GENERATION_MAX_TOKENS) -
- safetyMargin;
- budget -= approxTokens(buildChatSystemPrompt(systemPrompt));
- budget -= approxTokens(promptText);
+ 256;
+ const budget =
+ inputBudget -
+ approxTokens(buildChatSystemPrompt(systemPrompt)) -
+ approxTokens(promptText);
if (budget <= 0) return [];
- const selected: LlmMessage[] = [];
- let used = 0;
-
- for (let idx = trimmedCandidates.length - 1; idx >= 0; idx -= 1) {
- const message = trimmedCandidates[idx];
- if (!message) continue;
+ const messages = trimmedCandidates.map((message): LlmMessage => {
const isUser = message.sender === "self";
let text = isUser
? message.text
@@ -2212,28 +2208,36 @@ const Page: React.FC = () => {
}
}
- const cost = approxTokens(text);
+ return { role: isUser ? "user" : "assistant", content: text };
+ });
- if (used + cost > budget) {
- if (selected.length === 0 && budget > 0) {
- const charBudget = Math.max(1, budget * 4);
- const truncated = text.slice(-charBudget);
- selected.push({
- role: isUser ? "user" : "assistant",
- content: truncated,
- });
- }
- break;
- }
+ const historyTokens = messages.reduce(
+ (total, message) => total + approxTokens(message.content),
+ 0,
+ );
+ const quantum = Math.max(1, Math.floor(inputBudget / 4));
+ const overflow = Math.max(0, historyTokens - budget);
+ const discardTarget = Math.ceil(overflow / quantum) * quantum;
+ let discarded = 0;
+ let startIndex = 0;
+ while (startIndex < messages.length && discarded < discardTarget) {
+ discarded += approxTokens(messages[startIndex]!.content);
+ startIndex += 1;
+ }
- selected.push({
- role: isUser ? "user" : "assistant",
- content: text,
- });
- used += cost;
+ const selected = messages.slice(startIndex);
+ if (selected.length === 0 && messages.length > 0) {
+ const last = messages[messages.length - 1]!;
+ if (approxTokens(last.content) <= budget) return [last];
+ return [
+ {
+ ...last,
+ content: last.content.slice(-Math.max(1, budget * 4)),
+ },
+ ];
}
- return selected.reverse();
+ return selected;
},
[approxTokens, slicePathUntil, stripHiddenParts, systemPrompt],
);