π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Prune chat history in token quanta
Manav Rathib53f2eb63789
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.ktindex a100c14990..f57c0fcb1b 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt@@ -1099,31 +1099,37 @@ internal class ChatStoreActions(val promptTokens = estimatePromptTokens(promptText, promptImageCount)val historyTokens = historyMessages.sumOf { estimateTokens(historyText(it)) }val inputTokens = systemTokens + promptTokens + historyTokens- var remaining = inputBudget - systemTokens - promptTokens+ val remaining = inputBudget - systemTokens - promptTokensif (remaining <= 0 || historyMessages.isEmpty()) {return HistorySelection(emptyList(), inputTokens, inputBudget, inputTokens > inputBudget)}- val selected = mutableListOf<LlmMessage>()- for (message in historyMessages.asReversed()) {+ val quantum = max(1, inputBudget / 4)+ val overflow = max(0, historyTokens - remaining)+ val quantaToDiscard = (overflow + quantum - 1) / quantum+ val discardTarget = quantaToDiscard * quantum+ var discarded = 0+ var startIndex = 0+ while (startIndex < historyMessages.size && discarded < discardTarget) {+ discarded += estimateTokens(historyText(historyMessages[startIndex]))+ startIndex+++ }++ val retained = historyMessages.drop(startIndex).ifEmpty {+ historyMessages.takeLast(1).filter { estimateTokens(historyText(it)) <= remaining }+ }++ val selected = retained.map { message ->val text = historyText(message)- val cost = estimateTokens(text)- if (cost <= remaining) {- selected.add(- LlmMessage(- text = text,- role = if (message.author == MessageAuthor.User) LlmMessageRole.User else LlmMessageRole.Assistant,- hasAttachments = message.attachments.isNotEmpty()- )- )- remaining -= cost- } else {- break- }+ LlmMessage(+ text = text,+ role = if (message.author == MessageAuthor.User) LlmMessageRole.User else LlmMessageRole.Assistant,+ hasAttachments = message.attachments.isNotEmpty()+ )}- return HistorySelection(selected.reversed(), inputTokens, inputBudget, inputTokens > inputBudget)+ return HistorySelection(selected, inputTokens, inputBudget, inputTokens > inputBudget)}private fun resolveGenerationLimits(target: LlmModelTarget): GenerationLimits {diff --git a/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swiftindex ebbcbc7ae5..1a7eb57a86 100644--- a/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift@@ -1847,25 +1847,34 @@ final class ChatViewModel: ObservableObject {total + estimateTokens(historyText(node))}let inputTokens = systemTokens + promptTokens + historyTokens- var remaining = inputBudget - systemTokens - promptTokens+ let remaining = inputBudget - systemTokens - promptTokensif remaining <= 0 || historyMessages.isEmpty {return HistorySelection(messages: [], inputTokens: inputTokens, inputBudget: inputBudget, wasTrimmed: inputTokens > inputBudget)}- var selected: [LlmMessage] = []- for node in historyMessages.reversed() {+ let quantum = max(1, inputBudget / 4)+ let overflow = max(0, historyTokens - remaining)+ let quantaToDiscard = (overflow + quantum - 1) / quantum+ let discardTarget = quantaToDiscard * quantum+ var discarded = 0+ var startIndex = historyMessages.startIndex+ while startIndex < historyMessages.endIndex && discarded < discardTarget {+ discarded += estimateTokens(historyText(historyMessages[startIndex]))+ startIndex = historyMessages.index(after: startIndex)+ }++ var retained = historyMessages[startIndex...]+ if retained.isEmpty, let last = historyMessages.last, estimateTokens(historyText(last)) <= remaining {+ retained = historyMessages.suffix(1)+ }++ let selected = retained.map { node inlet text = historyText(node)- let cost = estimateTokens(text)- if cost <= remaining {- selected.append(LlmMessage(text: text, role: node.role == .user ? .user : .assistant, hasAttachments: !node.attachments.isEmpty))- remaining -= cost- } else {- break- }+ return LlmMessage(text: text, role: node.role == .user ? .user : .assistant, hasAttachments: !node.attachments.isEmpty)}- return HistorySelection(messages: selected.reversed(), inputTokens: inputTokens, inputBudget: inputBudget, wasTrimmed: inputTokens > inputBudget)+ return HistorySelection(messages: selected, inputTokens: inputTokens, inputBudget: inputBudget, wasTrimmed: inputTokens > inputBudget)}private func resolveGenerationLimits(target: LlmModelTarget) -> GenerationLimits {diff --git a/web/apps/ensu/src/pages/chat.tsx b/web/apps/ensu/src/pages/chat.tsxindex db237fa9a8..c7e294a3a1 100644--- a/web/apps/ensu/src/pages/chat.tsx+++ b/web/apps/ensu/src/pages/chat.tsx@@ -2179,22 +2179,18 @@ const Page: React.FC = () => {? candidates.slice(0, -1): candidates;- const safetyMargin = 256;- let budget =+ const inputBudget =contextSize -(maxTokensCount ?? DEFAULT_GENERATION_MAX_TOKENS) -- safetyMargin;- budget -= approxTokens(buildChatSystemPrompt(systemPrompt));- budget -= approxTokens(promptText);+ 256;+ const budget =+ inputBudget -+ approxTokens(buildChatSystemPrompt(systemPrompt)) -+ approxTokens(promptText);if (budget <= 0) return [];- const selected: LlmMessage[] = [];- let used = 0;-- for (let idx = trimmedCandidates.length - 1; idx >= 0; idx -= 1) {- const message = trimmedCandidates[idx];- if (!message) continue;+ const messages = trimmedCandidates.map((message): LlmMessage => {const isUser = message.sender === "self";let text = isUser? message.text@@ -2212,28 +2208,36 @@ const Page: React.FC = () => {}}- const cost = approxTokens(text);+ return { role: isUser ? "user" : "assistant", content: text };+ });- if (used + cost > budget) {- if (selected.length === 0 && budget > 0) {- const charBudget = Math.max(1, budget * 4);- const truncated = text.slice(-charBudget);- selected.push({- role: isUser ? "user" : "assistant",- content: truncated,- });- }- break;- }+ const historyTokens = messages.reduce(+ (total, message) => total + approxTokens(message.content),+ 0,+ );+ const quantum = Math.max(1, Math.floor(inputBudget / 4));+ const overflow = Math.max(0, historyTokens - budget);+ const discardTarget = Math.ceil(overflow / quantum) * quantum;+ let discarded = 0;+ let startIndex = 0;+ while (startIndex < messages.length && discarded < discardTarget) {+ discarded += approxTokens(messages[startIndex]!.content);+ startIndex += 1;+ }- selected.push({- role: isUser ? "user" : "assistant",- content: text,- });- used += cost;+ const selected = messages.slice(startIndex);+ if (selected.length === 0 && messages.length > 0) {+ const last = messages[messages.length - 1]!;+ if (approxTokens(last.content) <= budget) return [last];+ return [+ {+ ...last,+ content: last.content.slice(-Math.max(1, budget * 4)),+ },+ ];}- return selected.reverse();+ return selected;},[approxTokens, slicePathUntil, stripHiddenParts, systemPrompt],);