Skip to content

The Unified and Autonomous Development Platform

feat: Improved PR review process for AI user (OD-2756)

Robin Shen56455e89c690
diff --git a/server-core/src/main/java/io/onedev/server/ai/taskchecker/NoopTaskChecker.java b/server-core/src/main/java/io/onedev/server/ai/taskchecker/NoopTaskChecker.java
index c79f0f9a0e..98528ee87a 100644
--- a/server-core/src/main/java/io/onedev/server/ai/taskchecker/NoopTaskChecker.java
+++ b/server-core/src/main/java/io/onedev/server/ai/taskchecker/NoopTaskChecker.java
@@ -1,17 +1,17 @@
package io.onedev.server.ai.taskchecker;
-import java.util.Map;
+import java.util.Set;
public class NoopTaskChecker implements TaskChecker {
@Override
- public String preToolCall(String toolName, Map<String, Integer> toolCallCounts) {
+ public String preToolCall(String toolName, Set<String> calledTools) {
return null;
}
@Override
- public String preTaskFinish(Map<String, Integer> toolCallCounts) {
- return null;
+ public boolean isResponseRequired(Set<String> calledTools) {
+ return true;
}
}
\ No newline at end of file
diff --git a/server-core/src/main/java/io/onedev/server/ai/taskchecker/PullRequestReviewTaskChecker.java b/server-core/src/main/java/io/onedev/server/ai/taskchecker/PullRequestReviewTaskChecker.java
index 06df7bfdca..c01201014e 100644
--- a/server-core/src/main/java/io/onedev/server/ai/taskchecker/PullRequestReviewTaskChecker.java
+++ b/server-core/src/main/java/io/onedev/server/ai/taskchecker/PullRequestReviewTaskChecker.java
@@ -1,29 +1,29 @@
package io.onedev.server.ai.taskchecker;
-import java.util.Map;
+import static io.onedev.server.model.PullRequest.APPROVE_TOOL_NAME;
+import static io.onedev.server.model.PullRequest.REQUEST_FOR_CHANGES_TOOL_NAME;
-import io.onedev.server.model.PullRequest;
+import java.util.Set;
public class PullRequestReviewTaskChecker implements TaskChecker {
@Override
- public String preToolCall(String toolName, Map<String, Integer> toolCallCounts) {
- if ((toolName.equals(PullRequest.APPROVE_TOOL_NAME) || toolName.equals(PullRequest.REQUEST_FOR_CHANGES_TOOL_NAME))
- && toolCallCounts.getOrDefault(toolName, 0) >= 1) {
- return "Tool '%s' should only be called once".formatted(toolName);
+ public String preToolCall(String toolName, Set<String> calledTools) {
+ if (toolName.equals(APPROVE_TOOL_NAME) || toolName.equals(REQUEST_FOR_CHANGES_TOOL_NAME)) {
+ if (calledTools.contains(APPROVE_TOOL_NAME))
+ return "You've already approved the pull request";
+ else if (calledTools.contains(REQUEST_FOR_CHANGES_TOOL_NAME))
+ return "You've already requested changes for the pull request";
+ else
+ return null;
} else {
return null;
}
}
@Override
- public String preTaskFinish(Map<String, Integer> toolCallCounts) {
- if (toolCallCounts.getOrDefault(PullRequest.APPROVE_TOOL_NAME, 0) == 0
- && toolCallCounts.getOrDefault(PullRequest.REQUEST_FOR_CHANGES_TOOL_NAME, 0) == 0) {
- return "You must call exactly one of the following tools: '%s' or '%s'".formatted(PullRequest.APPROVE_TOOL_NAME, PullRequest.REQUEST_FOR_CHANGES_TOOL_NAME);
- } else {
- return null;
- }
+ public boolean isResponseRequired(Set<String> calledTools) {
+ return !calledTools.contains(APPROVE_TOOL_NAME) && !calledTools.contains(REQUEST_FOR_CHANGES_TOOL_NAME);
}
}
\ No newline at end of file
diff --git a/server-core/src/main/java/io/onedev/server/ai/taskchecker/TaskChecker.java b/server-core/src/main/java/io/onedev/server/ai/taskchecker/TaskChecker.java
index 9ddb6a7b89..5524e97fdf 100644
--- a/server-core/src/main/java/io/onedev/server/ai/taskchecker/TaskChecker.java
+++ b/server-core/src/main/java/io/onedev/server/ai/taskchecker/TaskChecker.java
@@ -1,15 +1,14 @@
package io.onedev.server.ai.taskchecker;
-import java.util.Map;
+import java.util.Set;
import org.jspecify.annotations.Nullable;
public interface TaskChecker {
@Nullable
- String preToolCall(String toolName, Map<String, Integer> toolCallCounts);
+ String preToolCall(String toolName, Set<String> calledTools);
- @Nullable
- String preTaskFinish(Map<String, Integer> toolCallCounts);
+ boolean isResponseRequired(Set<String> calledTools);
}
\ No newline at end of file
diff --git a/server-core/src/main/java/io/onedev/server/model/PullRequest.java b/server-core/src/main/java/io/onedev/server/model/PullRequest.java
index 89c573fb93..cc5be83f12 100644
--- a/server-core/src/main/java/io/onedev/server/model/PullRequest.java
+++ b/server-core/src/main/java/io/onedev/server/model/PullRequest.java
@@ -81,6 +81,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import dev.langchain4j.agent.tool.ToolSpecification;
+import dev.langchain4j.model.chat.request.json.JsonObjectSchema;
import io.onedev.server.OneDev;
import io.onedev.server.ai.PullRequestHelper;
import io.onedev.server.ai.TaskTool;
@@ -1562,6 +1563,10 @@ public class PullRequest extends ProjectBelonging
.name(APPROVE_TOOL_NAME)
.description("Record an approval for current OneDev pull request. "
+ "Call this exactly once after you have finished reviewing and decided to approve.")
+ .parameters(JsonObjectSchema.builder()
+ .addStringProperty("reason").description("Reason explaining why you are approving this pull request. Make sure to quote relevant code snippets if applicable")
+ .required("reason")
+ .build())
.build();
}
@@ -1572,8 +1577,12 @@ public class PullRequest extends ProjectBelonging
if (!SecurityUtils.canReadCode(subject, request.getProject()) || user == null)
throw new UnauthorizedException();
+ if (arguments.get("reason") == null)
+ return new ToolExecutionResult(convertToJson(Map.of("successful", false, "failReason", "Argument 'reason' is required")), false);
+ var reason = arguments.get("reason").asText();
+
try {
- getPullRequestReviewService().review(user, request, true, null);
+ getPullRequestReviewService().review(user, request, true, reason);
return new ToolExecutionResult(convertToJson(Map.of("successful", true)), false);
} catch (NotAcceptableException e) {
var data = Map.of(
@@ -1592,6 +1601,10 @@ public class PullRequest extends ProjectBelonging
.name(REQUEST_FOR_CHANGES_TOOL_NAME)
.description("Record a request for changes for current OneDev pull request. "
+ "Call this exactly once after you have finished reviewing and decided changes are needed.")
+ .parameters(JsonObjectSchema.builder()
+ .addStringProperty("reason").description("Reason explaining why you are requesting changes for this pull request. Make sure to quote relevant code snippets if applicable")
+ .required("reason")
+ .build())
.build();
}
@@ -1602,8 +1615,12 @@ public class PullRequest extends ProjectBelonging
if (!SecurityUtils.canReadCode(subject, request.getProject()) || user == null)
throw new UnauthorizedException();
+ if (arguments.get("reason") == null)
+ return new ToolExecutionResult(convertToJson(Map.of("successful", false, "failReason", "Argument 'reason' is required")), false);
+ var reason = arguments.get("reason").asText();
+
try {
- getPullRequestReviewService().review(user, request, false, null);
+ getPullRequestReviewService().review(user, request, false, reason);
return new ToolExecutionResult(convertToJson(Map.of("successful", true)), false);
} catch (NotAcceptableException e) {
var data = Map.of(
diff --git a/server-core/src/main/java/io/onedev/server/notification/PullRequestNotificationManager.java b/server-core/src/main/java/io/onedev/server/notification/PullRequestNotificationManager.java
index 6eddb7a0b6..ca1b114be5 100644
--- a/server-core/src/main/java/io/onedev/server/notification/PullRequestNotificationManager.java
+++ b/server-core/src/main/java/io/onedev/server/notification/PullRequestNotificationManager.java
@@ -3,6 +3,7 @@ package io.onedev.server.notification;
import static io.onedev.server.notification.NotificationUtils.getEmailBody;
import static io.onedev.server.notification.NotificationUtils.isNotified;
+import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
@@ -298,19 +299,15 @@ public class PullRequestNotificationManager {
} else if (isAiEntitled(null, request, reviewer)) {
var task = new AiTask(
null,
- """
+ MessageFormat.format("""
Review current pull request for major issues (ignore styling/format/documentation issues) \
introduced in the change. Check full content of relevant files to understand the change \
if necessary. Check existing comments for conversation context. \
- You MUST record your final decision by calling exactly one of these tools, and call it \
- only once: `approvePullRequest` if you are satisfied with the change, or \
- `requestChangesForPullRequest` if you think it needs more work. If you are unsure, \
- call `requestChangesForPullRequest` and explain what is unclear. Do not call either \
- tool more than once. \
-
- After calling the decision tool, summarize found issues in your final response and for \
- each issue, make sure to quote relevant code snippets if applicable""",
+ Record your final decision by calling one of these tools, and call it \
+ only once: `{0}` if you are satisfied with the change, or `{1}` if you think it needs more work. \
+ If you are unsure, explain what is unclear via final response""",
+ PullRequest.APPROVE_TOOL_NAME, PullRequest.REQUEST_FOR_CHANGES_TOOL_NAME),
request.getTools(true),
new PullRequestReviewTaskChecker(),
new AddPullRequestComment(request.getId()));
diff --git a/server-core/src/main/java/io/onedev/server/service/impl/DefaultUserService.java b/server-core/src/main/java/io/onedev/server/service/impl/DefaultUserService.java
index 79cf30e548..403a60f347 100644
--- a/server-core/src/main/java/io/onedev/server/service/impl/DefaultUserService.java
+++ b/server-core/src/main/java/io/onedev/server/service/impl/DefaultUserService.java
@@ -9,7 +9,7 @@ import java.io.ObjectStreamException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
@@ -90,8 +90,6 @@ public class DefaultUserService extends BaseEntityService<User> implements UserS
private static final int EMPTY_RESPONSE_MAX_RETRIES = 3;
- private static final int TASK_FINISH_CHECK_MAX_RETRIES = 3;
-
private static final String EMPTY_RESPONSE_PROMPT =
"Your previous turn was empty. Please reply with your answer.";
@@ -645,10 +643,9 @@ public class DefaultUserService extends BaseEntityService<User> implements UserS
ToolUtils.filterDuplications(toolSpecifications);
- var toolCallCounts = new HashMap<String, Integer>();
+ var calledTools = new HashSet<String>();
var taskChecker = task.getTaskChecker();
var emptyResponseRetryCount = new AtomicInteger(0);
- var taskFinishCheckRetryCount = new AtomicInteger(0);
while (true) {
if (Thread.interrupted())
throw new InterruptedException();
@@ -660,31 +657,27 @@ public class DefaultUserService extends BaseEntityService<User> implements UserS
var aiMessage = response.aiMessage();
if (!aiMessage.hasToolExecutionRequests()) {
- var finishError = taskChecker.preTaskFinish(toolCallCounts);
- if (finishError != null
- && taskFinishCheckRetryCount.incrementAndGet() <= TASK_FINISH_CHECK_MAX_RETRIES) {
- if (StringUtils.isNotBlank(aiMessage.text()))
- messages.add(aiMessage);
- messages.add(new UserMessage(finishError));
- continue;
- }
- var text = aiMessage.text();
- if (StringUtils.isNotBlank(text)) {
- sessionService.run(() -> {
- task.getResponseHandler().onResponse(
- Preconditions.checkNotNull(SecurityUtils.getUser(subject)),
- text);
- });
- break;
- } else if (emptyResponseRetryCount.incrementAndGet() <= EMPTY_RESPONSE_MAX_RETRIES) {
- messages.add(new UserMessage(EMPTY_RESPONSE_PROMPT));
- continue;
+ if (taskChecker.isResponseRequired(calledTools)) {
+ var responseText = aiMessage.text();
+ if (StringUtils.isNotBlank(responseText)) {
+ sessionService.run(() -> {
+ task.getResponseHandler().onResponse(
+ Preconditions.checkNotNull(SecurityUtils.getUser(subject)),
+ responseText);
+ });
+ break;
+ } else if (emptyResponseRetryCount.incrementAndGet() <= EMPTY_RESPONSE_MAX_RETRIES) {
+ messages.add(new UserMessage(EMPTY_RESPONSE_PROMPT));
+ continue;
+ } else {
+ sessionService.run(() -> {
+ task.getResponseHandler().onResponse(
+ Preconditions.checkNotNull(SecurityUtils.getUser(subject)),
+ "Empty response received");
+ });
+ break;
+ }
} else {
- sessionService.run(() -> {
- task.getResponseHandler().onResponse(
- Preconditions.checkNotNull(SecurityUtils.getUser(subject)),
- "Empty response received");
- });
break;
}
}
@@ -696,7 +689,7 @@ public class DefaultUserService extends BaseEntityService<User> implements UserS
throw new RuntimeException(new InterruptedException());
var toolName = toolRequest.name();
- var errorMessage = taskChecker.preToolCall(toolName, toolCallCounts);
+ var errorMessage = taskChecker.preToolCall(toolName, calledTools);
if (errorMessage != null) {
messages.add(ToolExecutionResultMessage.from(toolRequest.id(), toolName, errorMessage));
continue;
@@ -707,7 +700,7 @@ public class DefaultUserService extends BaseEntityService<User> implements UserS
.findFirst()
.orElseThrow(() -> new ExplicitException("Tool not found: " + toolName));
tool.execute(subject, getToolArguments(toolRequest)).addToMessages(messages, toolRequest);
- toolCallCounts.put(toolName, toolCallCounts.getOrDefault(toolName, 0) + 1);
+ calledTools.add(toolName);
} catch (Throwable t) {
ToolUtils.handleCallException(toolName, t);
}