Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[ensu] Derive reply budgets from context length

setalpfdf335287b9c
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
index 0167e8dc7e..4ac3e26490 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatStoreActions.kt
@@ -22,6 +22,7 @@ import io.ente.ensu.llm.LlmMessageRole
import io.ente.ensu.llm.LlmModelSelection
import io.ente.ensu.llm.LlmProvider
import io.ente.ensu.llm.ModelSettingsActions
+import io.ente.ensu.llm.automaticMaxOutputTokens
import io.ente.ensu.logging.FileLogRepository
import io.ente.ensu.logging.LogLevel
import io.ente.ensu.notes.NotesStore
@@ -850,9 +851,6 @@ internal class ChatStoreActions(
try {
llmProvider.withConversationContext(selection) { context ->
if (!isActive() || stopRequested) return@withConversationContext
- require(selection.maxTokens == null || selection.maxTokens > 0) {
- "Max output must be a positive whole number, or Auto"
- }
val preparation =
chatRepository.prepareConversation(
context,
@@ -863,7 +861,7 @@ internal class ChatStoreActions(
current = current,
expectedUserText = userMessage.text,
historyQuery = userMessage.text,
- maxTokens = selection.maxTokens?.toUInt(),
+ maxTokens = null,
searched = searched,
),
)
@@ -1584,14 +1582,14 @@ internal class ChatStoreActions(
llmProvider.loadedContextLength(selection)
?: selection.contextLength
?: DEFAULT_CONTEXT_LENGTH
- val maxOutput = resolveMaxOutputTokens(selection.maxTokens, contextLength)
+ val maxOutput = resolveMaxOutputTokens(contextLength)
return GenerationLimits(contextLength = contextLength, maxOutput = maxOutput)
}
- private fun resolveMaxOutputTokens(configuredMaxTokens: Int?, contextLength: Int): Int {
+ private fun resolveMaxOutputTokens(contextLength: Int): Int {
val maxAllowed = max(1, contextLength - OVERFLOW_SAFETY_TOKENS)
- val implicitMax = min(DEFAULT_GENERATION_MAX_TOKENS, max(1, contextLength / 2))
- return (configuredMaxTokens ?: implicitMax).coerceIn(1, maxAllowed)
+ val implicitMax = automaticMaxOutputTokens(contextLength)
+ return min(implicitMax, maxAllowed)
}
private fun showOverflowDialog(selection: HistorySelection, limits: GenerationLimits) {
@@ -1727,7 +1725,6 @@ internal class ChatStoreActions(
companion object {
private const val DEFAULT_CONTEXT_LENGTH = 12_000
- private const val DEFAULT_GENERATION_MAX_TOKENS = 8_192
private const val MEDIA_MARKER = "<__media__>"
private const val OVERFLOW_SAFETY_TOKENS = 256
private const val IMAGE_TOKEN_ESTIMATE = 768
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmModels.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmModels.kt
index e6cae66950..5c741a6f38 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmModels.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmModels.kt
@@ -1,9 +1,11 @@
package io.ente.ensu.llm
+internal fun automaticMaxOutputTokens(contextLength: Int): Int =
+ minOf(2048, maxOf(1, contextLength / 4))
+
data class LlmModelSelection(
val id: String,
val contextLength: Int? = null,
- val maxTokens: Int? = null,
)
data class DownloadProgress(
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
index dd7a7392da..45d5bd8897 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt
@@ -372,12 +372,10 @@ internal class ModelSettingsActions(
preferredModelId = settings.modelId.takeIf { it.isNotEmpty() },
)
val contextLength = settings.contextLength.toIntOrNull()
- val maxTokens = settings.maxTokens.toIntOrNull()?.takeIf { it > 0 }
return LlmModelSelection(
id = modelId,
contextLength = contextLength,
- maxTokens = maxTokens,
)
}
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsScreen.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsScreen.kt
index 5426797042..d8da7bda88 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsScreen.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsScreen.kt
@@ -3,13 +3,11 @@ package io.ente.ensu.llm
import android.widget.Toast
import androidx.compose.animation.AnimatedVisibility
import androidx.compose.foundation.layout.Column
-import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
-import androidx.compose.foundation.layout.width
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.text.KeyboardOptions
import androidx.compose.foundation.verticalScroll
@@ -75,15 +73,22 @@ fun ModelSettingsScreen(
var selectedModelId by
remember(state) { mutableStateOf(initialSelectionId(state, modelChoices)) }
var contextLength by remember(state) { mutableStateOf(state.contextLength) }
- var maxTokens by remember(state) { mutableStateOf(state.maxTokens) }
var temperature by remember(state) { mutableStateOf(state.temperature) }
+ val contextError =
+ contextLength
+ .takeIf { it.isNotBlank() }
+ ?.let { raw ->
+ val value = raw.toIntOrNull()
+ when {
+ value == null -> "Enter a valid integer"
+ value <= 256 + automaticMaxOutputTokens(value) ->
+ "Increase context length to leave room for the conversation and response."
+ else -> null
+ }
+ }
var showAdvancedLimits by
remember(state) {
- mutableStateOf(
- state.contextLength.isNotBlank() ||
- state.maxTokens.isNotBlank() ||
- state.temperature.isNotBlank()
- )
+ mutableStateOf(state.contextLength.isNotBlank() || state.temperature.isNotBlank())
}
var isModelMenuExpanded by remember { mutableStateOf(false) }
@@ -139,45 +144,32 @@ fun ModelSettingsScreen(
ExpandButton(
title = "Advanced limits",
expanded = showAdvancedLimits,
- collapsedHint = "Context length, output, temperature",
+ collapsedHint = "Context length and temperature",
onToggle = { showAdvancedLimits = !showAdvancedLimits },
)
AnimatedVisibility(showAdvancedLimits) {
Column {
Spacer(modifier = Modifier.height(EnsuSpacing.sm.dp))
- Row {
- Column(modifier = Modifier.weight(1f)) {
- Text(
- text = "Context length",
- style = EnsuTypography.small,
- color = EnsuColor.textMuted(),
- )
- Spacer(modifier = Modifier.height(EnsuSpacing.xs.dp))
- OutlinedTextField(
- value = contextLength,
- onValueChange = { contextLength = it },
- placeholder = { Text(text = "8192") },
- modifier = Modifier.fillMaxWidth(),
- keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Number),
- )
- }
- Spacer(modifier = Modifier.width(EnsuSpacing.md.dp))
- Column(modifier = Modifier.weight(1f)) {
- Text(
- text = "Max output",
- style = EnsuTypography.small,
- color = EnsuColor.textMuted(),
- )
- Spacer(modifier = Modifier.height(EnsuSpacing.xs.dp))
- OutlinedTextField(
- value = maxTokens,
- onValueChange = { maxTokens = it },
- placeholder = { Text(text = "2048") },
- modifier = Modifier.fillMaxWidth(),
- keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Number),
- )
- }
- }
+ Text(
+ text = "Context length",
+ style = EnsuTypography.small,
+ color = EnsuColor.textMuted(),
+ )
+ Spacer(modifier = Modifier.height(EnsuSpacing.xs.dp))
+ OutlinedTextField(
+ value = contextLength,
+ onValueChange = { contextLength = it },
+ placeholder = { Text(text = "8192") },
+ modifier = Modifier.fillMaxWidth(),
+ isError = contextError != null,
+ supportingText = { contextError?.let { Text(it) } },
+ keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Number),
+ )
+ Text(
+ text = "Response length adjusts automatically to the context length.",
+ style = EnsuTypography.small,
+ color = EnsuColor.textMuted(),
+ )
Spacer(modifier = Modifier.height(EnsuSpacing.md.dp))
Text(
@@ -214,12 +206,12 @@ fun ModelSettingsScreen(
Spacer(modifier = Modifier.height(EnsuSpacing.lg.dp))
Button(
+ enabled = contextError == null,
onClick = {
val savedState =
state.copy(
modelId = selectedModel.id.takeUnless { selectedModel.isDefault }.orEmpty(),
contextLength = contextLength,
- maxTokens = maxTokens,
temperature = temperature,
)
onSave(savedState)
@@ -238,7 +230,6 @@ fun ModelSettingsScreen(
onReset()
selectedModelId = DEFAULT_OPTION_ID
contextLength = ""
- maxTokens = ""
temperature = ""
Toast.makeText(context, "Model settings reset", Toast.LENGTH_SHORT).show()
}
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsState.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsState.kt
index 0d716081f6..f087e690a7 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsState.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsState.kt
@@ -3,7 +3,6 @@ package io.ente.ensu.llm
data class ModelSettingsState(
val modelId: String = "",
val contextLength: String = "",
- val maxTokens: String = "",
val temperature: String = "",
val isSaving: Boolean = false,
)
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/AdvancedSettingsDataStore.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/AdvancedSettingsDataStore.kt
index caf3af88a0..2765043b01 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/AdvancedSettingsDataStore.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/settings/AdvancedSettingsDataStore.kt
@@ -36,7 +36,6 @@ class AdvancedSettingsDataStore(private val context: Context) {
ModelSettingsState(
modelId = prefs[Keys.modelId].orEmpty(),
contextLength = prefs[Keys.contextLength].orEmpty(),
- maxTokens = prefs[Keys.maxTokens].orEmpty(),
temperature = prefs[Keys.temperature].orEmpty(),
),
)
@@ -59,6 +58,7 @@ class AdvancedSettingsDataStore(private val context: Context) {
prefs.remove(useCustomModel)
prefs.remove(modelUrl)
prefs.remove(mmprojUrl)
+ prefs.remove(Keys.legacyMaxTokens)
}
}
@@ -82,7 +82,7 @@ class AdvancedSettingsDataStore(private val context: Context) {
context.advancedSettingsPreferences.edit { prefs ->
prefs[Keys.modelId] = settings.modelId
prefs[Keys.contextLength] = settings.contextLength
- prefs[Keys.maxTokens] = settings.maxTokens
+ prefs.remove(Keys.legacyMaxTokens)
prefs[Keys.temperature] = settings.temperature
}
}
@@ -95,7 +95,7 @@ class AdvancedSettingsDataStore(private val context: Context) {
context.advancedSettingsPreferences.edit { prefs ->
prefs[Keys.modelId] = ""
prefs[Keys.contextLength] = ""
- prefs[Keys.maxTokens] = ""
+ prefs.remove(Keys.legacyMaxTokens)
prefs[Keys.temperature] = ""
}
}
@@ -110,7 +110,7 @@ class AdvancedSettingsDataStore(private val context: Context) {
val systemPrompt = stringPreferencesKey("system_prompt")
val modelId = stringPreferencesKey("model_id")
val contextLength = stringPreferencesKey("context_length")
- val maxTokens = stringPreferencesKey("max_tokens")
+ val legacyMaxTokens = stringPreferencesKey("max_tokens")
val temperature = stringPreferencesKey("temperature")
}
}
diff --git a/apple/apps/ensu/Ensu/Chat/ChatView.swift b/apple/apps/ensu/Ensu/Chat/ChatView.swift
index 6ff6342326..e9488c39af 100644
--- a/apple/apps/ensu/Ensu/Chat/ChatView.swift
+++ b/apple/apps/ensu/Ensu/Chat/ChatView.swift
@@ -47,7 +47,6 @@ struct ChatView: View {
[
modelSettings.modelId,
modelSettings.contextLength,
- modelSettings.maxTokens,
modelSettings.temperature,
modelSettings.systemPromptBody,
]
diff --git a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
index 2d84204a60..f1efecae80 100644
--- a/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
+++ b/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift
@@ -29,7 +29,6 @@ final class ChatViewModel: ObservableObject {
}()
private static let systemPromptDatePlaceholder = ConfigDefaults.shared
.systemPromptDatePlaceholder
- private static let defaultGenerationMaxTokens = 8_192
private static let overflowSafetyTokens = 256
private static let imageTokenEstimate = 768
private nonisolated static let sessionTitleMaxLength = 40
@@ -1319,7 +1318,7 @@ final class ChatViewModel: ObservableObject {
current: prompt.text,
expectedUserText: userNode.text,
historyQuery: userNode.text,
- maxTokens: selection.maxTokens.map { UInt32(clamping: $0) },
+ maxTokens: nil,
searched: knowledgeHits
),
temperature: self.resolveTemperature(),
@@ -2323,15 +2322,14 @@ final class ChatViewModel: ObservableObject {
private func resolveGenerationLimits(_ selection: LlmModelSelection) async -> GenerationLimits {
let contextLength =
await provider.loadedContextLength(selection) ?? selection.contextLength ?? 12000
- let maxOutput = resolveMaxOutputTokens(
- configuredMaxTokens: selection.maxTokens, contextLength: contextLength)
+ let maxOutput = resolveMaxOutputTokens(contextLength: contextLength)
return GenerationLimits(contextLength: contextLength, maxOutput: maxOutput)
}
- private func resolveMaxOutputTokens(configuredMaxTokens: Int?, contextLength: Int) -> Int {
+ private func resolveMaxOutputTokens(contextLength: Int) -> Int {
let maxAllowed = max(1, contextLength - Self.overflowSafetyTokens)
- let implicitMax = min(Self.defaultGenerationMaxTokens, max(1, contextLength / 2))
- return min(maxAllowed, configuredMaxTokens ?? implicitMax)
+ let implicitMax = automaticMaxOutputTokens(contextLength: contextLength)
+ return min(maxAllowed, implicitMax)
}
private func historyText(_ node: MessageNode) -> String {
diff --git a/apple/apps/ensu/Ensu/ModelSettings/ModelSettingsView.swift b/apple/apps/ensu/Ensu/ModelSettings/ModelSettingsView.swift
index 76ed81ef02..36a4fb1f16 100644
--- a/apple/apps/ensu/Ensu/ModelSettings/ModelSettingsView.swift
+++ b/apple/apps/ensu/Ensu/ModelSettings/ModelSettingsView.swift
@@ -8,11 +8,9 @@ struct ModelSettingsView: View {
@ObservedObject private var settings = ModelSettingsStore.shared
@State private var selectedModelId: String = defaultOptionId
@State private var contextLength: String = ""
- @State private var maxTokens: String = ""
@State private var temperature: String = ""
@State private var contextError: String?
- @State private var maxTokensError: String?
@State private var temperatureError: String?
@State private var isSaving = false
@State private var showAdvancedLimits = false
@@ -66,11 +64,9 @@ struct ModelSettingsView: View {
.onAppear {
selectedModelId = initialSelectionId()
contextLength = settings.contextLength
- maxTokens = settings.maxTokens
temperature = settings.temperature
showAdvancedLimits =
- !settings.contextLength.isEmpty || !settings.maxTokens.isEmpty
- || !settings.temperature.isEmpty
+ !settings.contextLength.isEmpty || !settings.temperature.isEmpty
}
.overlay(alignment: .bottom) {
if let toastMessage {
@@ -104,7 +100,7 @@ struct ModelSettingsView: View {
sectionToggle(
title: "Advanced limits",
- collapsedHint: "Context length, output, temperature",
+ collapsedHint: "Context length and temperature",
expanded: showAdvancedLimits
) {
showAdvancedLimits.toggle()
@@ -112,23 +108,18 @@ struct ModelSettingsView: View {
if showAdvancedLimits {
VStack(spacing: EnsuSpacing.sm) {
- HStack(spacing: EnsuSpacing.md) {
- field(
- label: "Context length",
- hint: "8192",
- text: $contextLength,
- error: contextError,
- keyboardType: .numberPad
- )
-
- field(
- label: "Max output",
- hint: "2048",
- text: $maxTokens,
- error: maxTokensError,
- keyboardType: .numberPad
- )
- }
+ field(
+ label: "Context length",
+ hint: "8192",
+ text: $contextLength,
+ error: contextError,
+ keyboardType: .numberPad
+ )
+
+ Text("Response length adjusts automatically to the context length.")
+ .font(EnsuTypography.small)
+ .foregroundStyle(EnsuColor.textMuted)
+ .frame(maxWidth: .infinity, alignment: .leading)
Text("Leave blank to use model defaults")
.font(EnsuTypography.small)
@@ -233,7 +224,6 @@ struct ModelSettingsView: View {
private func saveTapped() {
contextError = nil
- maxTokensError = nil
temperatureError = nil
guard validate() else { return }
@@ -244,7 +234,6 @@ struct ModelSettingsView: View {
settings.saveModel(
id: selectedModel.isDefault ? "" : selectedModel.id,
contextLength: contextLength,
- maxTokens: maxTokens,
temperature: temperature
)
isSaving = false
@@ -259,7 +248,6 @@ struct ModelSettingsView: View {
settings.resetToDefault()
selectedModelId = Self.defaultOptionId
contextLength = ""
- maxTokens = ""
temperature = ""
toastTask?.cancel()
toastTask = presentToast("Model settings reset") { message in
@@ -270,14 +258,17 @@ struct ModelSettingsView: View {
private func validate() -> Bool {
var isValid = true
- if !contextLength.isEmpty, Int(contextLength) == nil {
- contextError = "Enter a valid integer"
- isValid = false
- }
-
- if !maxTokens.isEmpty, Int(maxTokens) == nil {
- maxTokensError = "Enter a valid integer"
- isValid = false
+ if !contextLength.isEmpty {
+ if let value = Int32(contextLength) {
+ if Int(value) <= 256 + automaticMaxOutputTokens(contextLength: Int(value)) {
+ contextError =
+ "Increase context length to leave room for the conversation and response."
+ isValid = false
+ }
+ } else {
+ contextError = "Enter a valid integer"
+ isValid = false
+ }
}
if !temperature.isEmpty, Float(temperature) == nil {
@@ -285,13 +276,6 @@ struct ModelSettingsView: View {
isValid = false
}
- if let contextValue = Int(contextLength), let maxValue = Int(maxTokens),
- maxValue > contextValue
- {
- maxTokensError = "Must be <= context length"
- isValid = false
- }
-
return isValid
}
diff --git a/apple/apps/ensu/Ensu/Services/LlmProvider.swift b/apple/apps/ensu/Ensu/Services/LlmProvider.swift
index 4aa75f4b24..c395155999 100644
--- a/apple/apps/ensu/Ensu/Services/LlmProvider.swift
+++ b/apple/apps/ensu/Ensu/Services/LlmProvider.swift
@@ -1,10 +1,13 @@
import Foundation
import os
+func automaticMaxOutputTokens(contextLength: Int) -> Int {
+ min(2048, max(1, contextLength / 4))
+}
+
struct LlmModelSelection: Equatable {
let id: String
let contextLength: Int?
- let maxTokens: Int?
}
struct DownloadProgress: Equatable {
diff --git a/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift b/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift
index eb9dbfcba1..03595a46a5 100644
--- a/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift
+++ b/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift
@@ -58,9 +58,6 @@ final class ModelSettingsStore: ObservableObject {
@Published var contextLength: String {
didSet { persist() }
}
- @Published var maxTokens: String {
- didSet { persist() }
- }
@Published var temperature: String {
didSet { persist() }
}
@@ -73,22 +70,20 @@ final class ModelSettingsStore: ObservableObject {
private init() {
self.modelId = defaults.string(forKey: Keys.modelId) ?? ""
self.contextLength = defaults.string(forKey: Keys.contextLength) ?? ""
- self.maxTokens = defaults.string(forKey: Keys.maxTokens) ?? ""
self.temperature = defaults.string(forKey: Keys.temperature) ?? ""
self.systemPromptBody = defaults.string(forKey: Keys.systemPromptBody) ?? ""
+ defaults.removeObject(forKey: Keys.legacyMaxTokens)
}
- func saveModel(id: String, contextLength: String, maxTokens: String, temperature: String) {
+ func saveModel(id: String, contextLength: String, temperature: String) {
modelId = id
self.contextLength = contextLength
- self.maxTokens = maxTokens
self.temperature = temperature
}
func resetToDefault() {
modelId = ""
contextLength = ""
- maxTokens = ""
temperature = ""
}
@@ -100,8 +95,7 @@ final class ModelSettingsStore: ObservableObject {
)
return LlmModelSelection(
id: modelId,
- contextLength: Int(contextLength),
- maxTokens: Int(maxTokens).flatMap { $0 > 0 ? $0 : nil }
+ contextLength: Int(contextLength)
)
}
@@ -129,7 +123,6 @@ final class ModelSettingsStore: ObservableObject {
private func persist() {
defaults.set(modelId, forKey: Keys.modelId)
defaults.set(contextLength, forKey: Keys.contextLength)
- defaults.set(maxTokens, forKey: Keys.maxTokens)
defaults.set(temperature, forKey: Keys.temperature)
defaults.set(systemPromptBody, forKey: Keys.systemPromptBody)
}
@@ -137,7 +130,7 @@ final class ModelSettingsStore: ObservableObject {
fileprivate enum Keys {
static let modelId = "ensu.model.id"
static let contextLength = "ensu.model.context"
- static let maxTokens = "ensu.model.max_tokens"
+ static let legacyMaxTokens = "ensu.model.max_tokens"
static let temperature = "ensu.model.temperature"
static let systemPromptBody = "ensu.model.system_prompt_body"
}
diff --git a/rust/crates/ensu/src/conversation/prepare.rs b/rust/crates/ensu/src/conversation/prepare.rs
index 576461e6dc..2dcca0e95c 100644
--- a/rust/crates/ensu/src/conversation/prepare.rs
+++ b/rust/crates/ensu/src/conversation/prepare.rs
@@ -85,14 +85,17 @@ impl Preparation {
});
let state = state.filter(|_| covered.is_some());
let covered = covered.unwrap_or(0);
- let summary_output = SUMMARY_OUTPUT_TOKENS.min(context / 4);
+ let answer_input_budget = input_budget(context, output)?;
+ let summary_output = SUMMARY_OUTPUT_TOKENS
+ .min(context / 4)
+ .min(context - SAFETY_TOKENS - 1);
Ok(Self {
session,
history,
system,
history_query: Some(current.clone()),
current,
- input_budget: input_budget(context, output)?,
+ input_budget: answer_input_budget,
summary_input_budget: input_budget(context, summary_output)?,
summary_output,
state,
diff --git a/web/apps/ensu/src/components/chat/ChatDialogs.tsx b/web/apps/ensu/src/components/chat/ChatDialogs.tsx
index a8bb1677f0..ca8992b98e 100644
--- a/web/apps/ensu/src/components/chat/ChatDialogs.tsx
+++ b/web/apps/ensu/src/components/chat/ChatDialogs.tsx
@@ -1,6 +1,7 @@
import type { KnowledgePack } from "@/services/knowledge";
import {
DEFAULT_TAURI_CONTEXT_SIZE,
+ DEFAULT_WEB_CONTEXT_SIZE,
resolveGenerationBudget,
} from "@/services/llm/budget";
import type { NotesCollectionView } from "@/services/notes-lifecycle";
@@ -98,7 +99,6 @@ const compactInlineLinkSx = {
export interface ModelSettingsDraft {
modelId: string;
contextLength: string;
- maxTokens: string;
}
export interface ChatDialogsProps {
@@ -138,7 +138,6 @@ export interface ChatDialogsProps {
isTauriRuntime: boolean;
suggestedModels: SuggestedModel[];
contextLength: string;
- maxTokens: string;
isSavingModel: boolean;
handleSaveModel: (draft: ModelSettingsDraft) => void;
handleUseDefaultModel: () => void;
@@ -210,7 +209,6 @@ export const ChatDialogs = memo(
isTauriRuntime,
suggestedModels,
contextLength,
- maxTokens,
isSavingModel,
handleSaveModel,
handleUseDefaultModel,
@@ -264,7 +262,6 @@ export const ChatDialogs = memo(
};
const [draftContextLength, setDraftContextLength] = React.useState("");
- const [draftMaxTokens, setDraftMaxTokens] = React.useState("");
const isModelPreparationActive =
modelGateStatus === "checking" ||
modelGateStatus === "preloading" ||
@@ -272,9 +269,6 @@ export const ChatDialogs = memo(
const [draftContextError, setDraftContextError] = React.useState<
string | null
>(null);
- const [draftMaxTokensError, setDraftMaxTokensError] = React.useState<
- string | null
- >(null);
const [showAdvancedLimits, setShowAdvancedLimits] =
React.useState(false);
const [draftModelId, setDraftModelId] = React.useState("default");
@@ -311,17 +305,9 @@ export const ChatDialogs = memo(
: "default",
);
setDraftContextLength(contextLength);
- setDraftMaxTokens(maxTokens);
setDraftContextError(null);
- setDraftMaxTokensError(null);
- setShowAdvancedLimits(!!contextLength || !!maxTokens);
- }, [
- contextLength,
- maxTokens,
- modelOptions,
- selectedModelId,
- showModelSettings,
- ]);
+ setShowAdvancedLimits(!!contextLength);
+ }, [contextLength, modelOptions, selectedModelId, showModelSettings]);
React.useEffect(() => {
const didOpen =
@@ -336,48 +322,27 @@ export const ChatDialogs = memo(
draftContextLength && !/^\d+$/.test(draftContextLength)
? "Enter a number"
: undefined;
- let maxTokensErrorValue =
- draftMaxTokens && !/^\d+$/.test(draftMaxTokens)
- ? "Enter a number"
- : undefined;
const contextValue = draftContextLength
? Number(draftContextLength)
: undefined;
- const maxTokensValue = draftMaxTokens
- ? Number(draftMaxTokens)
- : undefined;
-
- if (isTauriRuntime) {
- const contextSize = contextValue ?? DEFAULT_TAURI_CONTEXT_SIZE;
- if (!contextErrorValue) {
- try {
- resolveGenerationBudget(contextSize, 1);
- } catch (error) {
- contextErrorValue = (error as Error).message;
- }
- }
- if (!contextErrorValue && !maxTokensErrorValue) {
- try {
- resolveGenerationBudget(contextSize, maxTokensValue);
- } catch (error) {
- maxTokensErrorValue = (error as Error).message;
- }
+ if (!contextErrorValue) {
+ const contextSize = isTauriRuntime
+ ? (contextValue ?? DEFAULT_TAURI_CONTEXT_SIZE)
+ : Math.min(
+ contextValue ?? DEFAULT_WEB_CONTEXT_SIZE,
+ DEFAULT_WEB_CONTEXT_SIZE,
+ );
+ try {
+ resolveGenerationBudget(contextSize);
+ } catch (error) {
+ contextErrorValue = (error as Error).message;
}
- } else if (
- !maxTokensErrorValue &&
- contextValue &&
- maxTokensValue &&
- maxTokensValue > contextValue
- ) {
- maxTokensErrorValue = "Must be <= context length";
}
setDraftContextError(contextErrorValue ?? null);
- setDraftMaxTokensError(maxTokensErrorValue ?? null);
-
- return !(contextErrorValue || maxTokensErrorValue);
- }, [draftContextLength, draftMaxTokens, isTauriRuntime]);
+ return !contextErrorValue;
+ }, [draftContextLength, isTauriRuntime]);
return (
<>
@@ -1512,7 +1477,7 @@ export const ChatDialogs = memo(
variant="mini"
sx={{ color: "text.muted" }}
>
- Context length and max output
+ Context length
</Typography>
)}
{showAdvancedLimits && (
@@ -1535,34 +1500,15 @@ export const ChatDialogs = memo(
draftContextError ?? " "
}
/>
- <TextField
- fullWidth
- label="Max output"
- placeholder={
- isTauriRuntime
- ? "Auto"
- : "2048"
- }
- value={draftMaxTokens}
- onChange={(event) =>
- setDraftMaxTokens(
- event.target.value,
- )
- }
- error={!!draftMaxTokensError}
- helperText={
- draftMaxTokensError ?? " "
- }
- />
</Stack>
)}
<Typography
variant="mini"
sx={{ color: "text.muted" }}
>
- {isTauriRuntime
- ? "Leave blank for Auto. Output adjusts to the available context."
- : "Leave blank to use model defaults."}
+ Leave blank to use model defaults.
+ Response length adjusts automatically to
+ the context length.
</Typography>
</Stack>
</Stack>
@@ -1584,7 +1530,6 @@ export const ChatDialogs = memo(
? ""
: draftModelId,
contextLength: draftContextLength,
- maxTokens: draftMaxTokens,
});
}}
>
diff --git a/web/apps/ensu/src/pages/chat.tsx b/web/apps/ensu/src/pages/chat.tsx
index 4e4ddc7842..4d7ef2c6ff 100644
--- a/web/apps/ensu/src/pages/chat.tsx
+++ b/web/apps/ensu/src/pages/chat.tsx
@@ -575,7 +575,6 @@ const Page: React.FC = () => {
>(null);
const [selectedModelId, setSelectedModelId] = useState("");
const [contextLength, setContextLength] = useState("");
- const [maxTokens, setMaxTokens] = useState("");
const [systemPrompt, setSystemPrompt] = useState(
DEFAULT_CHAT_SYSTEM_PROMPT_BODY,
);
@@ -1048,7 +1047,6 @@ const Page: React.FC = () => {
const applySettings = (parsed: {
modelId?: string;
contextLength?: string;
- maxTokens?: string;
}) => {
const rawContextLength = parsed.contextLength ?? "";
const clampedContextLength =
@@ -1060,7 +1058,6 @@ const Page: React.FC = () => {
const settings = {
modelId: parsed.modelId ?? "",
contextLength: clampedContextLength,
- maxTokens: parsed.maxTokens ?? "",
};
window.localStorage.setItem(
MODEL_SETTINGS_STORAGE_KEY,
@@ -1069,7 +1066,6 @@ const Page: React.FC = () => {
if (cancelled) return;
setSelectedModelId(settings.modelId);
setContextLength(settings.contextLength);
- setMaxTokens(settings.maxTokens);
};
// Older builds persisted a model URL selection; llm_migrate_models
@@ -1097,7 +1093,6 @@ const Page: React.FC = () => {
modelUrl?: string;
mmprojUrl?: string;
contextLength?: string;
- maxTokens?: string;
})
: {};
let modelId = parsed.modelId ?? "";
@@ -1122,11 +1117,7 @@ const Page: React.FC = () => {
modelId = converted ?? "";
}
}
- applySettings({
- modelId,
- contextLength: parsed.contextLength,
- maxTokens: parsed.maxTokens,
- });
+ applySettings({ modelId, contextLength: parsed.contextLength });
};
void loadSettings()
@@ -1869,12 +1860,8 @@ const Page: React.FC = () => {
return {
modelId: selectedModelId || undefined,
contextLength: contextLength ? Number(contextLength) : undefined,
- maxTokens:
- maxTokens && (isTauriRuntime || Number(maxTokens) > 0)
- ? Number(maxTokens)
- : undefined,
};
- }, [selectedModelId, contextLength, maxTokens, isTauriRuntime]);
+ }, [selectedModelId, contextLength]);
const modelSettingsKey = useMemo(
() => JSON.stringify(getModelSettings()),
@@ -1883,7 +1870,7 @@ const Page: React.FC = () => {
const formatErrorMessage = useCallback((error: unknown) => {
if (isNamedError(error, "prompt_too_long")) {
- return "Prompt exceeds the model context window. Reduce history, lower max tokens, or increase context length.";
+ return "Prompt exceeds the model context window. Reduce history or increase context length.";
}
return tauriCommandError(error).message ?? "Unknown model error";
}, []);
@@ -3069,7 +3056,7 @@ const Page: React.FC = () => {
normalPromptTokenEstimate > inputBudget
) {
throw new Error(
- "Prompt exceeds the model context window. Reduce history, lower max tokens, or increase context length.",
+ "Prompt exceeds the model context window. Reduce history or increase context length.",
);
}
@@ -3187,7 +3174,7 @@ const Page: React.FC = () => {
);
if (countTokens(normalMessages) > inputBudget) {
throw new Error(
- "Prompt exceeds the loaded model context window. Reduce history, lower max output, or increase context length.",
+ "Prompt exceeds the loaded model context window. Reduce history or increase context length.",
);
}
}
@@ -3799,11 +3786,7 @@ const Page: React.FC = () => {
modelGateStatus === "downloading";
const handleSaveModel = useCallback(
- (draft: {
- modelId: string;
- contextLength: string;
- maxTokens: string;
- }) => {
+ (draft: { modelId: string; contextLength: string }) => {
if (isModelPreparationActive) return;
setIsSavingModel(true);
if (typeof window !== "undefined") {
@@ -3814,7 +3797,6 @@ const Page: React.FC = () => {
}
setSelectedModelId(draft.modelId);
setContextLength(draft.contextLength);
- setMaxTokens(draft.maxTokens);
modelGateRequestRef.current += 1;
setLoadedModelName(null);
setModelDownloadSizeBytes(null);
@@ -3830,16 +3812,11 @@ const Page: React.FC = () => {
if (typeof window !== "undefined") {
window.localStorage.setItem(
MODEL_SETTINGS_STORAGE_KEY,
- JSON.stringify({
- modelId: "",
- contextLength: "",
- maxTokens: "",
- }),
+ JSON.stringify({ modelId: "", contextLength: "" }),
);
}
setSelectedModelId("");
setContextLength("");
- setMaxTokens("");
modelGateRequestRef.current += 1;
setLoadedModelName(null);
setModelDownloadSizeBytes(null);
@@ -4823,7 +4800,6 @@ const Page: React.FC = () => {
isTauriRuntime={isTauriRuntime}
suggestedModels={suggestedModels}
contextLength={contextLength}
- maxTokens={maxTokens}
isSavingModel={isSavingModel}
handleSaveModel={handleSaveModel}
handleUseDefaultModel={handleUseDefaultModel}
diff --git a/web/apps/ensu/src/services/llm/budget.ts b/web/apps/ensu/src/services/llm/budget.ts
index 09cbf88058..5eb4121e0d 100644
--- a/web/apps/ensu/src/services/llm/budget.ts
+++ b/web/apps/ensu/src/services/llm/budget.ts
@@ -19,14 +19,12 @@ export function resolveGenerationBudget(
const maxTokens =
configuredMaxTokens ?? Math.min(2048, Math.floor(contextSize / 4));
if (!Number.isSafeInteger(maxTokens) || maxTokens <= 0) {
- throw new Error(
- "Max output must be a positive whole number, or blank for Auto.",
- );
+ throw new Error("The response budget must be a positive whole number.");
}
const inputBudget = contextSize - maxTokens - OVERFLOW_SAFETY_TOKENS;
if (inputBudget <= 0) {
throw new Error(
- `Max output leaves no room for a prompt in the ${contextSize}-token context. Lower max output, choose Auto, or increase context length.`,
+ `The ${contextSize}-token context cannot fit the conversation and response. Increase context length.`,
);
}
return { contextSize, maxTokens, inputBudget };
diff --git a/web/apps/ensu/src/services/llm/provider.ts b/web/apps/ensu/src/services/llm/provider.ts
index ba66db35f1..14dd511e72 100644
--- a/web/apps/ensu/src/services/llm/provider.ts
+++ b/web/apps/ensu/src/services/llm/provider.ts
@@ -234,20 +234,6 @@ export class LlmProvider {
this.backend.kind === "tauri"
? requestedContextSize
: Math.min(requestedContextSize, DEFAULT_WEB_CONTEXT_SIZE);
- const configuredMaxTokens = settings.maxTokens ?? model.maxTokens;
- if (this.backend.kind !== "tauri") {
- const maxTokens = Math.min(
- configuredMaxTokens ??
- Math.min(8_192, Math.max(1, Math.floor(contextSize / 2))),
- Math.max(1, contextSize - 256),
- );
- return {
- model,
- contextSize,
- maxTokens,
- inputBudget: contextSize - maxTokens - 256,
- };
- }
resolveGenerationBudget(contextSize, 1);
const loadedContextSize =
useLoadedContext &&
@@ -258,10 +244,7 @@ export class LlmProvider {
: undefined;
return {
model,
- ...resolveGenerationBudget(
- loadedContextSize ?? contextSize,
- configuredMaxTokens,
- ),
+ ...resolveGenerationBudget(loadedContextSize ?? contextSize),
};
}
diff --git a/web/apps/ensu/src/services/llm/types.ts b/web/apps/ensu/src/services/llm/types.ts
index c5ac08d4f8..10bb99a6b5 100644
--- a/web/apps/ensu/src/services/llm/types.ts
+++ b/web/apps/ensu/src/services/llm/types.ts
@@ -16,13 +16,11 @@ export interface ModelInfo {
sizeBytes?: number;
mmprojSizeBytes?: number;
contextLength?: number;
- maxTokens?: number;
}
export interface ModelSettings {
modelId?: string;
contextLength?: number;
- maxTokens?: number;
}
export interface DownloadProgress {