The Unified and Autonomous Development Platform
fix: AI Chat - Content gets thrown away after response streaming finishes (OD-2717)
Robin Shen36dae71b34bd
diff --git a/server-core/src/main/java/io/onedev/server/ai/DefaultChatService.java b/server-core/src/main/java/io/onedev/server/ai/DefaultChatService.javaindex ebefce2e86..e394ce1269 100644--- a/server-core/src/main/java/io/onedev/server/ai/DefaultChatService.java+++ b/server-core/src/main/java/io/onedev/server/ai/DefaultChatService.java@@ -76,7 +76,6 @@ import io.onedev.server.taskschedule.TaskScheduler;import io.onedev.server.util.concurrent.BatchWorkExecutionService;import io.onedev.server.util.concurrent.BatchWorker;import io.onedev.server.util.concurrent.Prioritized;-import io.onedev.server.ai.tools.TaskComplete;import io.onedev.server.web.SessionListener;import io.onedev.server.web.WebSession;import io.onedev.server.web.page.base.BasePage;@@ -94,6 +93,11 @@ public class DefaultChatService extends BaseEntityService<Chat> implements ChatSprivate static final int MAX_HISTORY_MESSAGE_LEN = 1024;+ private static final int EMPTY_RESPONSE_MAX_RETRIES = 3;++ private static final String EMPTY_RESPONSE_PROMPT =+ "Your previous turn was empty. Please reply with your answer to the user.";+private static final int HOUSE_KEEPING_PRIORITY = 50;@Inject@@ -163,9 +167,6 @@ public class DefaultChatService extends BaseEntityService<Chat> implements ChatS.stream().map(ChatTool::getSpecification).collect(Collectors.toList());-- // Add the taskComplete tool - this must always be present- toolSpecifications.add(TaskComplete.getSpecification());ToolUtils.filterDuplications(toolSpecifications);@@ -194,7 +195,6 @@ public class DefaultChatService extends BaseEntityService<Chat> implements ChatSvar systemPrompt = request.getChat().getAi().getAiSetting().getSystemPrompt();if (systemPrompt != null)langchain4jMessages.add(new SystemMessage(systemPrompt));- langchain4jMessages.add(new SystemMessage(TaskComplete.SYSTEM_MESSAGE));langchain4jMessages.addAll(messages.stream().map(it -> {var content = it.getContent();@@ -210,7 +210,7 @@ public class DefaultChatService extends BaseEntityService<Chat> implements ChatSvar responseFuture = new CompletableFuture<String>();var executionFuture = executorService.submit(new Runnable() {- private int noToolRetryCount = 0;+ private int emptyResponseRetryCount = 0;private StreamingChatResponseHandler newResponseHandler(Subject subject) {return new StreamingChatResponseHandler() {@@ -241,7 +241,9 @@ public class DefaultChatService extends BaseEntityService<Chat> implements ChatS@Overridepublic void onPartialThinking(PartialThinking partialThinking, PartialThinkingContext context) {- updateRespondingContent(context.streamingHandle(), partialThinking.text());+ ThreadContext.bind(subject);+ if (responseFuture.isDone())+ context.streamingHandle().cancel();}@Override@@ -286,14 +288,6 @@ public class DefaultChatService extends BaseEntityService<Chat> implements ChatSif (responseFuture.isDone())return;String toolName = toolRequest.name();-- // Handle taskComplete specially - it signals conversation completion- var taskCompleteResponse = TaskComplete.getResponse(toolRequest);- if (taskCompleteResponse != null) {- responseFuture.complete(taskCompleteResponse);- return;- }-try {var toolExecution = new AiToolExecution(toolName, ToolUtils.getToolArguments(toolRequest));connection.sendMessage(toolExecution);@@ -328,15 +322,20 @@ public class DefaultChatService extends BaseEntityService<Chat> implements ChatS.build();streamingChatModel.chat(langchain4jChatRequest, handler);} else {- // LLM didn't call any tool - prompt it to continue or call taskComplete- noToolRetryCount++;- if (noToolRetryCount <= TaskComplete.MAX_NO_TOOL_RETRIES) {- var text = aiMessage.text();- if (StringUtils.isNotBlank(text)) {- langchain4jMessages.add(aiMessage);- }- langchain4jMessages.add(new UserMessage(TaskComplete.CONTINUATION_PROMPT));-+ // No tool calls. If the assistant produced text, use it+ // directly so the persisted response matches what the user+ // already saw on the stream. We deliberately do NOT route the+ // final answer through a tool argument, because models tend to+ // summarize when re-emitting their answer that way and the+ // preview ends up richer than the saved response.+ var text = aiMessage.text();+ if (StringUtils.isNotBlank(text)) {+ responseFuture.complete(text);+ } else if (emptyResponseRetryCount < EMPTY_RESPONSE_MAX_RETRIES) {+ // Some models occasionally finish with no text and no+ // tool call. Nudge them to actually answer.+ emptyResponseRetryCount++;+ langchain4jMessages.add(new UserMessage(EMPTY_RESPONSE_PROMPT));var handler = newResponseHandler(SecurityUtils.getSubject());var langchain4jChatRequest = ChatRequest.builder().messages(new ArrayList<>(langchain4jMessages))@@ -344,8 +343,7 @@ public class DefaultChatService extends BaseEntityService<Chat> implements ChatS.build();streamingChatModel.chat(langchain4jChatRequest, handler);} else {- // Max retries exceeded, accept whatever response we have- responseFuture.complete(aiMessage.text());+ responseFuture.complete("");}}} catch (Throwable t) {diff --git a/server-core/src/main/java/io/onedev/server/ai/tools/TaskComplete.java b/server-core/src/main/java/io/onedev/server/ai/tools/TaskComplete.javadeleted file mode 100644index 053acceab9..0000000000--- a/server-core/src/main/java/io/onedev/server/ai/tools/TaskComplete.java+++ /dev/null@@ -1,58 +0,0 @@-package io.onedev.server.ai.tools;--import org.jspecify.annotations.Nullable;--import dev.langchain4j.agent.tool.ToolExecutionRequest;-import dev.langchain4j.agent.tool.ToolSpecification;-import dev.langchain4j.model.chat.request.json.JsonObjectSchema;-import io.onedev.commons.utils.StringUtils;-import io.onedev.server.ai.ToolUtils;--/**- * Special tool that signals the AI has completed its task and is ready to provide- * the final response. This tool is handled specially by DefaultChatService - when- * called, it completes the conversation with the provided response instead of- * continuing the tool execution loop.- */-public class TaskComplete {-- public static final String TOOL_NAME = "taskComplete";-- public static final int MAX_NO_TOOL_RETRIES = 3;-- public static final String SYSTEM_MESSAGE =- "IMPORTANT: When you have completed your task, you MUST call the 'taskComplete' tool with your final response. " +- "Do not provide a response without calling a tool. Always either call a tool to perform an action, or call 'taskComplete' to finish.";-- public static final String CONTINUATION_PROMPT =- "Please continue with your task, or call 'taskComplete' tool with your final response if you are finished.";-- public static ToolSpecification getSpecification() {- return ToolSpecification.builder()- .name(TOOL_NAME)- .description("Call this tool when you have completed the task and are ready to provide your final response to the user. " +- "You MUST call this tool to finish the conversation - do not respond without calling a tool.")- .parameters(JsonObjectSchema.builder()- .addStringProperty("response", "Your final response to the user's request. This will be shown directly to the user.")- .required("response")- .build())- .build();- }-- /**- * Checks if the tool request is a taskComplete call and extracts the response.- *- * @param toolRequest the tool execution request- * @return the response string if this is a taskComplete call, null otherwise- */- @Nullable- public static String getResponse(ToolExecutionRequest toolRequest) {- if (TOOL_NAME.equals(toolRequest.name())) {- var arguments = ToolUtils.getToolArguments(toolRequest);- var responseNode = arguments.get("response");- return responseNode != null ? StringUtils.trimToEmpty(responseNode.asText()) : "";- }- return null;- }--}diff --git a/server-core/src/main/java/io/onedev/server/service/impl/DefaultUserService.java b/server-core/src/main/java/io/onedev/server/service/impl/DefaultUserService.javaindex ef16526bba..8aeff0e75c 100644--- a/server-core/src/main/java/io/onedev/server/service/impl/DefaultUserService.java+++ b/server-core/src/main/java/io/onedev/server/service/impl/DefaultUserService.java@@ -10,7 +10,6 @@ import java.util.Collection;import java.util.List;import java.util.UUID;import java.util.concurrent.ExecutorService;-import java.util.concurrent.atomic.AtomicBoolean;import java.util.concurrent.atomic.AtomicInteger;import java.util.stream.Collectors;@@ -43,7 +42,6 @@ import io.onedev.commons.utils.StringUtils;import io.onedev.server.ai.AiTask;import io.onedev.server.ai.TaskTool;import io.onedev.server.ai.ToolUtils;-import io.onedev.server.ai.tools.TaskComplete;import io.onedev.server.cluster.ClusterService;import io.onedev.server.event.Listen;import io.onedev.server.event.cluster.NodeStarted;@@ -85,6 +83,11 @@ public class DefaultUserService extends BaseEntityService<User> implements UserSprivate static final int TIMEOUT_SECONDS = 600;+ private static final int EMPTY_RESPONSE_MAX_RETRIES = 3;++ private static final String EMPTY_RESPONSE_PROMPT =+ "Your previous turn was empty. Please reply with your answer.";+private static final int SYNC_PRIORITY = 50;@Inject@@ -627,19 +630,15 @@ public class DefaultUserService extends BaseEntityService<User> implements UserSmessages.add(new SystemMessage(aiSetting.getSystemPrompt()));if (task.getSystemPrompt() != null)messages.add(new SystemMessage(task.getSystemPrompt()));- messages.add(new SystemMessage(TaskComplete.SYSTEM_MESSAGE));messages.add(new UserMessage(task.getUserPrompt()));var tools = task.getTools();var toolSpecifications = tools.stream().map(TaskTool::getSpecification).collect(Collectors.toList());-- // Add the taskComplete tool - this must always be present- toolSpecifications.add(TaskComplete.getSpecification());-+ToolUtils.filterDuplications(toolSpecifications);-- var noToolRetryCount = new AtomicInteger(0);++ var emptyResponseRetryCount = new AtomicInteger(0);while (true) {if (Thread.interrupted())throw new InterruptedException();@@ -651,42 +650,39 @@ public class DefaultUserService extends BaseEntityService<User> implements UserSvar aiMessage = response.aiMessage();if (!aiMessage.hasToolExecutionRequests()) {- // LLM didn't call any tool - prompt it to continue or call taskComplete+ // No tool calls means the assistant has produced its final answer.+ // Use it directly instead of routing it through a tool argument+ // (which tends to make models summarize and lose detail).var text = aiMessage.text();- if (noToolRetryCount.incrementAndGet() <= TaskComplete.MAX_NO_TOOL_RETRIES) {- if (StringUtils.isNotBlank(text))- messages.add(aiMessage);- messages.add(new UserMessage(TaskComplete.CONTINUATION_PROMPT));+ if (StringUtils.isNotBlank(text)) {+ sessionService.run(() -> {+ task.getResponseHandler().onResponse(+ Preconditions.checkNotNull(SecurityUtils.getUser(subject)),+ text);+ });+ break;+ } else if (emptyResponseRetryCount.incrementAndGet() <= EMPTY_RESPONSE_MAX_RETRIES) {+ // Some models occasionally finish with no text and no tool+ // call. Nudge them to actually answer.+ messages.add(new UserMessage(EMPTY_RESPONSE_PROMPT));continue;} else {- // Max retries exceeded, accept whatever response we havesessionService.run(() -> {task.getResponseHandler().onResponse(- Preconditions.checkNotNull(SecurityUtils.getUser(subject)),- StringUtils.isBlank(text)?"Empty response received":text);+ Preconditions.checkNotNull(SecurityUtils.getUser(subject)),+ "Empty response received");});break;}}-+messages.add(aiMessage);- var taskCompleted = new AtomicBoolean(false);sessionService.run(() -> {for (var toolRequest : aiMessage.toolExecutionRequests()) {if (Thread.interrupted())- throw new RuntimeException(new InterruptedException());-- // Handle taskComplete specially - it signals task completion- var taskCompleteResponse = TaskComplete.getResponse(toolRequest);- if (taskCompleteResponse != null) {- task.getResponseHandler().onResponse(- Preconditions.checkNotNull(SecurityUtils.getUser(subject)),- StringUtils.isBlank(taskCompleteResponse)?"Empty response received":taskCompleteResponse);- taskCompleted.set(true);- return;- }+ throw new RuntimeException(new InterruptedException());- var toolName = toolRequest.name();+ var toolName = toolRequest.name();try {var tool = tools.stream().filter(it -> it.getSpecification().name().equals(toolName))@@ -698,9 +694,6 @@ public class DefaultUserService extends BaseEntityService<User> implements UserS}}});-- if (taskCompleted.get())- break;}} catch (Throwable e) {sessionService.run(() -> {