Skip to content

The Unified and Autonomous Development Platform

fix: AI Chat - Content gets thrown away after response streaming finishes (OD-2717)

Robin Shen36dae71b34bd
diff --git a/server-core/src/main/java/io/onedev/server/ai/DefaultChatService.java b/server-core/src/main/java/io/onedev/server/ai/DefaultChatService.java
index ebefce2e86..e394ce1269 100644
--- a/server-core/src/main/java/io/onedev/server/ai/DefaultChatService.java
+++ b/server-core/src/main/java/io/onedev/server/ai/DefaultChatService.java
@@ -76,7 +76,6 @@ import io.onedev.server.taskschedule.TaskScheduler;
import io.onedev.server.util.concurrent.BatchWorkExecutionService;
import io.onedev.server.util.concurrent.BatchWorker;
import io.onedev.server.util.concurrent.Prioritized;
-import io.onedev.server.ai.tools.TaskComplete;
import io.onedev.server.web.SessionListener;
import io.onedev.server.web.WebSession;
import io.onedev.server.web.page.base.BasePage;
@@ -94,6 +93,11 @@ public class DefaultChatService extends BaseEntityService<Chat> implements ChatS
private static final int MAX_HISTORY_MESSAGE_LEN = 1024;
+ private static final int EMPTY_RESPONSE_MAX_RETRIES = 3;
+
+ private static final String EMPTY_RESPONSE_PROMPT =
+ "Your previous turn was empty. Please reply with your answer to the user.";
+
private static final int HOUSE_KEEPING_PRIORITY = 50;
@Inject
@@ -163,9 +167,6 @@ public class DefaultChatService extends BaseEntityService<Chat> implements ChatS
.stream()
.map(ChatTool::getSpecification)
.collect(Collectors.toList());
-
- // Add the taskComplete tool - this must always be present
- toolSpecifications.add(TaskComplete.getSpecification());
ToolUtils.filterDuplications(toolSpecifications);
@@ -194,7 +195,6 @@ public class DefaultChatService extends BaseEntityService<Chat> implements ChatS
var systemPrompt = request.getChat().getAi().getAiSetting().getSystemPrompt();
if (systemPrompt != null)
langchain4jMessages.add(new SystemMessage(systemPrompt));
- langchain4jMessages.add(new SystemMessage(TaskComplete.SYSTEM_MESSAGE));
langchain4jMessages.addAll(messages.stream()
.map(it -> {
var content = it.getContent();
@@ -210,7 +210,7 @@ public class DefaultChatService extends BaseEntityService<Chat> implements ChatS
var responseFuture = new CompletableFuture<String>();
var executionFuture = executorService.submit(new Runnable() {
- private int noToolRetryCount = 0;
+ private int emptyResponseRetryCount = 0;
private StreamingChatResponseHandler newResponseHandler(Subject subject) {
return new StreamingChatResponseHandler() {
@@ -241,7 +241,9 @@ public class DefaultChatService extends BaseEntityService<Chat> implements ChatS
@Override
public void onPartialThinking(PartialThinking partialThinking, PartialThinkingContext context) {
- updateRespondingContent(context.streamingHandle(), partialThinking.text());
+ ThreadContext.bind(subject);
+ if (responseFuture.isDone())
+ context.streamingHandle().cancel();
}
@Override
@@ -286,14 +288,6 @@ public class DefaultChatService extends BaseEntityService<Chat> implements ChatS
if (responseFuture.isDone())
return;
String toolName = toolRequest.name();
-
- // Handle taskComplete specially - it signals conversation completion
- var taskCompleteResponse = TaskComplete.getResponse(toolRequest);
- if (taskCompleteResponse != null) {
- responseFuture.complete(taskCompleteResponse);
- return;
- }
-
try {
var toolExecution = new AiToolExecution(toolName, ToolUtils.getToolArguments(toolRequest));
connection.sendMessage(toolExecution);
@@ -328,15 +322,20 @@ public class DefaultChatService extends BaseEntityService<Chat> implements ChatS
.build();
streamingChatModel.chat(langchain4jChatRequest, handler);
} else {
- // LLM didn't call any tool - prompt it to continue or call taskComplete
- noToolRetryCount++;
- if (noToolRetryCount <= TaskComplete.MAX_NO_TOOL_RETRIES) {
- var text = aiMessage.text();
- if (StringUtils.isNotBlank(text)) {
- langchain4jMessages.add(aiMessage);
- }
- langchain4jMessages.add(new UserMessage(TaskComplete.CONTINUATION_PROMPT));
-
+ // No tool calls. If the assistant produced text, use it
+ // directly so the persisted response matches what the user
+ // already saw on the stream. We deliberately do NOT route the
+ // final answer through a tool argument, because models tend to
+ // summarize when re-emitting their answer that way and the
+ // preview ends up richer than the saved response.
+ var text = aiMessage.text();
+ if (StringUtils.isNotBlank(text)) {
+ responseFuture.complete(text);
+ } else if (emptyResponseRetryCount < EMPTY_RESPONSE_MAX_RETRIES) {
+ // Some models occasionally finish with no text and no
+ // tool call. Nudge them to actually answer.
+ emptyResponseRetryCount++;
+ langchain4jMessages.add(new UserMessage(EMPTY_RESPONSE_PROMPT));
var handler = newResponseHandler(SecurityUtils.getSubject());
var langchain4jChatRequest = ChatRequest.builder()
.messages(new ArrayList<>(langchain4jMessages))
@@ -344,8 +343,7 @@ public class DefaultChatService extends BaseEntityService<Chat> implements ChatS
.build();
streamingChatModel.chat(langchain4jChatRequest, handler);
} else {
- // Max retries exceeded, accept whatever response we have
- responseFuture.complete(aiMessage.text());
+ responseFuture.complete("");
}
}
} catch (Throwable t) {
diff --git a/server-core/src/main/java/io/onedev/server/ai/tools/TaskComplete.java b/server-core/src/main/java/io/onedev/server/ai/tools/TaskComplete.java
deleted file mode 100644
index 053acceab9..0000000000
--- a/server-core/src/main/java/io/onedev/server/ai/tools/TaskComplete.java
+++ /dev/null
@@ -1,58 +0,0 @@
-package io.onedev.server.ai.tools;
-
-import org.jspecify.annotations.Nullable;
-
-import dev.langchain4j.agent.tool.ToolExecutionRequest;
-import dev.langchain4j.agent.tool.ToolSpecification;
-import dev.langchain4j.model.chat.request.json.JsonObjectSchema;
-import io.onedev.commons.utils.StringUtils;
-import io.onedev.server.ai.ToolUtils;
-
-/**
- * Special tool that signals the AI has completed its task and is ready to provide
- * the final response. This tool is handled specially by DefaultChatService - when
- * called, it completes the conversation with the provided response instead of
- * continuing the tool execution loop.
- */
-public class TaskComplete {
-
- public static final String TOOL_NAME = "taskComplete";
-
- public static final int MAX_NO_TOOL_RETRIES = 3;
-
- public static final String SYSTEM_MESSAGE =
- "IMPORTANT: When you have completed your task, you MUST call the 'taskComplete' tool with your final response. " +
- "Do not provide a response without calling a tool. Always either call a tool to perform an action, or call 'taskComplete' to finish.";
-
- public static final String CONTINUATION_PROMPT =
- "Please continue with your task, or call 'taskComplete' tool with your final response if you are finished.";
-
- public static ToolSpecification getSpecification() {
- return ToolSpecification.builder()
- .name(TOOL_NAME)
- .description("Call this tool when you have completed the task and are ready to provide your final response to the user. " +
- "You MUST call this tool to finish the conversation - do not respond without calling a tool.")
- .parameters(JsonObjectSchema.builder()
- .addStringProperty("response", "Your final response to the user's request. This will be shown directly to the user.")
- .required("response")
- .build())
- .build();
- }
-
- /**
- * Checks if the tool request is a taskComplete call and extracts the response.
- *
- * @param toolRequest the tool execution request
- * @return the response string if this is a taskComplete call, null otherwise
- */
- @Nullable
- public static String getResponse(ToolExecutionRequest toolRequest) {
- if (TOOL_NAME.equals(toolRequest.name())) {
- var arguments = ToolUtils.getToolArguments(toolRequest);
- var responseNode = arguments.get("response");
- return responseNode != null ? StringUtils.trimToEmpty(responseNode.asText()) : "";
- }
- return null;
- }
-
-}
diff --git a/server-core/src/main/java/io/onedev/server/service/impl/DefaultUserService.java b/server-core/src/main/java/io/onedev/server/service/impl/DefaultUserService.java
index ef16526bba..8aeff0e75c 100644
--- a/server-core/src/main/java/io/onedev/server/service/impl/DefaultUserService.java
+++ b/server-core/src/main/java/io/onedev/server/service/impl/DefaultUserService.java
@@ -10,7 +10,6 @@ import java.util.Collection;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@@ -43,7 +42,6 @@ import io.onedev.commons.utils.StringUtils;
import io.onedev.server.ai.AiTask;
import io.onedev.server.ai.TaskTool;
import io.onedev.server.ai.ToolUtils;
-import io.onedev.server.ai.tools.TaskComplete;
import io.onedev.server.cluster.ClusterService;
import io.onedev.server.event.Listen;
import io.onedev.server.event.cluster.NodeStarted;
@@ -85,6 +83,11 @@ public class DefaultUserService extends BaseEntityService<User> implements UserS
private static final int TIMEOUT_SECONDS = 600;
+ private static final int EMPTY_RESPONSE_MAX_RETRIES = 3;
+
+ private static final String EMPTY_RESPONSE_PROMPT =
+ "Your previous turn was empty. Please reply with your answer.";
+
private static final int SYNC_PRIORITY = 50;
@Inject
@@ -627,19 +630,15 @@ public class DefaultUserService extends BaseEntityService<User> implements UserS
messages.add(new SystemMessage(aiSetting.getSystemPrompt()));
if (task.getSystemPrompt() != null)
messages.add(new SystemMessage(task.getSystemPrompt()));
- messages.add(new SystemMessage(TaskComplete.SYSTEM_MESSAGE));
messages.add(new UserMessage(task.getUserPrompt()));
var tools = task.getTools();
var toolSpecifications = tools.stream()
.map(TaskTool::getSpecification)
.collect(Collectors.toList());
-
- // Add the taskComplete tool - this must always be present
- toolSpecifications.add(TaskComplete.getSpecification());
-
+
ToolUtils.filterDuplications(toolSpecifications);
-
- var noToolRetryCount = new AtomicInteger(0);
+
+ var emptyResponseRetryCount = new AtomicInteger(0);
while (true) {
if (Thread.interrupted())
throw new InterruptedException();
@@ -651,42 +650,39 @@ public class DefaultUserService extends BaseEntityService<User> implements UserS
var aiMessage = response.aiMessage();
if (!aiMessage.hasToolExecutionRequests()) {
- // LLM didn't call any tool - prompt it to continue or call taskComplete
+ // No tool calls means the assistant has produced its final answer.
+ // Use it directly instead of routing it through a tool argument
+ // (which tends to make models summarize and lose detail).
var text = aiMessage.text();
- if (noToolRetryCount.incrementAndGet() <= TaskComplete.MAX_NO_TOOL_RETRIES) {
- if (StringUtils.isNotBlank(text))
- messages.add(aiMessage);
- messages.add(new UserMessage(TaskComplete.CONTINUATION_PROMPT));
+ if (StringUtils.isNotBlank(text)) {
+ sessionService.run(() -> {
+ task.getResponseHandler().onResponse(
+ Preconditions.checkNotNull(SecurityUtils.getUser(subject)),
+ text);
+ });
+ break;
+ } else if (emptyResponseRetryCount.incrementAndGet() <= EMPTY_RESPONSE_MAX_RETRIES) {
+ // Some models occasionally finish with no text and no tool
+ // call. Nudge them to actually answer.
+ messages.add(new UserMessage(EMPTY_RESPONSE_PROMPT));
continue;
} else {
- // Max retries exceeded, accept whatever response we have
sessionService.run(() -> {
task.getResponseHandler().onResponse(
- Preconditions.checkNotNull(SecurityUtils.getUser(subject)),
- StringUtils.isBlank(text)?"Empty response received":text);
+ Preconditions.checkNotNull(SecurityUtils.getUser(subject)),
+ "Empty response received");
});
break;
}
}
-
+
messages.add(aiMessage);
- var taskCompleted = new AtomicBoolean(false);
sessionService.run(() -> {
for (var toolRequest : aiMessage.toolExecutionRequests()) {
if (Thread.interrupted())
- throw new RuntimeException(new InterruptedException());
-
- // Handle taskComplete specially - it signals task completion
- var taskCompleteResponse = TaskComplete.getResponse(toolRequest);
- if (taskCompleteResponse != null) {
- task.getResponseHandler().onResponse(
- Preconditions.checkNotNull(SecurityUtils.getUser(subject)),
- StringUtils.isBlank(taskCompleteResponse)?"Empty response received":taskCompleteResponse);
- taskCompleted.set(true);
- return;
- }
+ throw new RuntimeException(new InterruptedException());
- var toolName = toolRequest.name();
+ var toolName = toolRequest.name();
try {
var tool = tools.stream()
.filter(it -> it.getSpecification().name().equals(toolName))
@@ -698,9 +694,6 @@ public class DefaultUserService extends BaseEntityService<User> implements UserS
}
}
});
-
- if (taskCompleted.get())
- break;
}
} catch (Throwable e) {
sessionService.run(() -> {