Skip to content

The Unified and Autonomous Development Platform

refactor: Better error handling for uploadpack/receivepack/advertiseuploadrefs/advertisereceiverefs

Robin Shen76d17420452a
diff --git a/server-core/src/main/java/io/onedev/server/cluster/ClusterResource.java b/server-core/src/main/java/io/onedev/server/cluster/ClusterResource.java
index 4d58443584..ad687dd34e 100644
--- a/server-core/src/main/java/io/onedev/server/cluster/ClusterResource.java
+++ b/server-core/src/main/java/io/onedev/server/cluster/ClusterResource.java
@@ -46,8 +46,6 @@ import io.onedev.server.codequality.UnitTestReport;
import io.onedev.server.git.GitFilter;
import io.onedev.server.git.GitUtils;
import io.onedev.server.git.LfsObject;
-import io.onedev.server.git.command.AdvertiseReceiveRefsCommand;
-import io.onedev.server.git.command.AdvertiseUploadRefsCommand;
import io.onedev.server.git.hook.HookUtils;
import io.onedev.server.model.PackBlob;
import io.onedev.server.model.Project;
@@ -275,9 +273,9 @@ public class ClusterResource {
StreamingOutput os = output -> {
File gitDir = projectService.getGitDir(projectId);
if (upload)
- new AdvertiseUploadRefsCommand(gitDir, output).protocol(protocol).run();
+ GitUtils.advertiseUploadRefs(gitDir, protocol, output);
else
- new AdvertiseReceiveRefsCommand(gitDir, output).protocol(protocol).run();
+ GitUtils.advertiseReceiveRefs(gitDir, protocol, output);
};
return ok(os).build();
}
diff --git a/server-core/src/main/java/io/onedev/server/git/GitFilter.java b/server-core/src/main/java/io/onedev/server/git/GitFilter.java
index 22499ecc64..0be34f586d 100644
--- a/server-core/src/main/java/io/onedev/server/git/GitFilter.java
+++ b/server-core/src/main/java/io/onedev/server/git/GitFilter.java
@@ -45,8 +45,6 @@ import io.onedev.k8shelper.KubernetesHelper;
import io.onedev.server.OneDev;
import io.onedev.server.cluster.ClusterService;
import io.onedev.server.exception.ServerNotReadyException;
-import io.onedev.server.git.command.AdvertiseReceiveRefsCommand;
-import io.onedev.server.git.command.AdvertiseUploadRefsCommand;
import io.onedev.server.git.hook.HookUtils;
import io.onedev.server.model.Project;
import io.onedev.server.persistence.SessionService;
@@ -337,10 +335,10 @@ public class GitFilter implements Filter {
String activeServer = projectService.getActiveServer(projectId, true);
if (activeServer.equals(clusterService.getLocalServerAddress())) {
File gitDir = projectService.getGitDir(projectId);
- if (upload)
- new AdvertiseUploadRefsCommand(gitDir, output).protocol(protocol).run();
- else
- new AdvertiseReceiveRefsCommand(gitDir, output).protocol(protocol).run();
+ if (upload)
+ GitUtils.advertiseUploadRefs(gitDir, protocol, output);
+ else
+ GitUtils.advertiseReceiveRefs(gitDir, protocol, output);
} else {
Client client = ClientBuilder.newClient();
try {
diff --git a/server-core/src/main/java/io/onedev/server/git/GitUtils.java b/server-core/src/main/java/io/onedev/server/git/GitUtils.java
index 76aaa126aa..1619544344 100644
--- a/server-core/src/main/java/io/onedev/server/git/GitUtils.java
+++ b/server-core/src/main/java/io/onedev/server/git/GitUtils.java
@@ -19,6 +19,7 @@ import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import org.apache.commons.lang3.Strings;
@@ -63,6 +64,7 @@ import org.eclipse.jgit.revwalk.RevTree;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.revwalk.RevWalkUtils;
import org.eclipse.jgit.revwalk.filter.RevFilter;
+import org.eclipse.jgit.transport.PacketLineOut;
import org.eclipse.jgit.treewalk.CanonicalTreeParser;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.treewalk.filter.TreeFilter;
@@ -93,6 +95,8 @@ import io.onedev.k8shelper.KubernetesHelper;
import io.onedev.server.OneDev;
import io.onedev.server.cluster.ClusterService;
import io.onedev.server.exception.NotFoundException;
+import io.onedev.server.git.command.AdvertiseReceiveRefsCommand;
+import io.onedev.server.git.command.AdvertiseUploadRefsCommand;
import io.onedev.server.git.command.FileChange;
import io.onedev.server.git.command.IsAncestorCommand;
import io.onedev.server.git.command.ReceivePackCommand;
@@ -857,9 +861,44 @@ public class GitUtils {
return change;
}
- public static void uploadPack(File gitDir, Map<String, String> environments, String protocol,
+ public static void advertiseUploadRefs(File gitDir, String protocol, OutputStream stdout) {
+ advertiseUploadRefs(gitDir, protocol, stdout, true);
+ }
+
+ public static void advertiseUploadRefs(File gitDir, String protocol, OutputStream stdout, boolean logError) {
+ AtomicBoolean outputWritten = new AtomicBoolean(false);
+ var errorMessage = new AtomicReference<String>();
+ var advertise = new AdvertiseUploadRefsCommand(gitDir, trackOutput(stdout, outputWritten),
+ errorLogger(errorMessage, logError));
+ var result = advertise.protocol(protocol).run();
+ handleGitResult(result, stdout, outputWritten, errorMessage.get(),
+ "git upload-pack --advertise-refs failed", logError);
+ }
+
+ public static void advertiseReceiveRefs(File gitDir, String protocol, OutputStream stdout) {
+ advertiseReceiveRefs(gitDir, protocol, stdout, true);
+ }
+
+ public static void advertiseReceiveRefs(File gitDir, String protocol, OutputStream stdout, boolean logError) {
+ AtomicBoolean outputWritten = new AtomicBoolean(false);
+ var errorMessage = new AtomicReference<String>();
+ var advertise = new AdvertiseReceiveRefsCommand(gitDir, trackOutput(stdout, outputWritten),
+ errorLogger(errorMessage, logError));
+ var result = advertise.protocol(protocol).run();
+ handleGitResult(result, stdout, outputWritten, errorMessage.get(),
+ "git receive-pack --advertise-refs failed", logError);
+ }
+
+ public static void uploadPack(File gitDir, Map<String, String> environments, String protocol,
InputStream stdin, OutputStream stdout) {
+ uploadPack(gitDir, environments, protocol, stdin, stdout, true);
+ }
+
+ public static void uploadPack(File gitDir, Map<String, String> environments, String protocol,
+ InputStream stdin, OutputStream stdout, boolean logError) {
AtomicBoolean toleratedErrors = new AtomicBoolean(false);
+ AtomicBoolean outputWritten = new AtomicBoolean(false);
+ var errorMessage = new AtomicReference<String>();
var stderr = new LineConsumer(UTF_8.name()) {
@Override
@@ -869,35 +908,93 @@ public class GitUtils {
toleratedErrors.set(true);
logger.debug(line);
} else {
- logger.error(line);
+ errorMessage.compareAndSet(null, line);
+ if (logError)
+ logger.error(line);
}
}
};
- ExecutionResult result;
- var upload = new UploadPackCommand(gitDir, stdin, stdout, stderr, environments);
+ var upload = new UploadPackCommand(gitDir, stdin, trackOutput(stdout, outputWritten), stderr, environments);
upload.statelessRpc(true).protocol(protocol);
- result = upload.run();
- if (result.getReturnCode() != 0 && !toleratedErrors.get())
- throw result.buildException();
+ var result = upload.run();
+ if (!toleratedErrors.get())
+ handleGitResult(result, stdout, outputWritten, errorMessage.get(), "git upload-pack failed", logError);
}
-
+
public static void receivePack(File gitDir, Map<String, String> environments, String protocol,
InputStream stdin, OutputStream stdout) {
- var stderr = new LineConsumer(UTF_8.name()) {
+ receivePack(gitDir, environments, protocol, stdin, stdout, true);
+ }
+
+ public static void receivePack(File gitDir, Map<String, String> environments, String protocol,
+ InputStream stdin, OutputStream stdout, boolean logError) {
+ AtomicBoolean outputWritten = new AtomicBoolean(false);
+ var errorMessage = new AtomicReference<String>();
+ var receive = new ReceivePackCommand(gitDir, stdin, trackOutput(stdout, outputWritten),
+ errorLogger(errorMessage, logError), environments);
+ receive.statelessRpc(true).protocol(protocol);
+ ExecutionResult result = receive.run();
+ handleGitResult(result, stdout, outputWritten, errorMessage.get(), "git receive-pack failed", logError);
+ }
+
+ private static LineConsumer errorLogger(AtomicReference<String> errorMessage, boolean logError) {
+ return new LineConsumer(UTF_8.name()) {
@Override
public void consume(String line) {
- logger.error(line);
+ errorMessage.compareAndSet(null, line);
+ if (logError)
+ logger.error(line);
}
-
+
};
-
- var receive = new ReceivePackCommand(gitDir, stdin, stdout, stderr, environments);
- receive.statelessRpc(true).protocol(protocol);
- ExecutionResult result = receive.run();
- result.checkReturnCode();
+ }
+
+ private static OutputStream trackOutput(OutputStream output, AtomicBoolean outputWritten) {
+ return new OutputStream() {
+
+ @Override
+ public void write(int b) throws IOException {
+ output.write(b);
+ outputWritten.set(true);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ output.write(b, off, len);
+ if (len != 0)
+ outputWritten.set(true);
+ }
+
+ @Override
+ public void flush() throws IOException {
+ output.flush();
+ }
+
+ };
+ }
+
+ private static void handleGitResult(ExecutionResult result, OutputStream output, AtomicBoolean outputWritten,
+ String errorMessage, String fallbackMessage, boolean logError) {
+ if (result.getReturnCode() != 0) {
+ if (!outputWritten.get()) {
+ var message = errorMessage;
+ if (message == null)
+ message = fallbackMessage;
+ try {
+ var packet = new PacketLineOut(output);
+ packet.writeString("ERR " + message + "\n");
+ packet.end();
+ output.flush();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ } else if (errorMessage == null && logError) {
+ logger.error("{} (exit code: {})", fallbackMessage, result.getReturnCode());
+ }
+ }
}
}
diff --git a/server-core/src/main/java/io/onedev/server/git/command/AdvertiseReceiveRefsCommand.java b/server-core/src/main/java/io/onedev/server/git/command/AdvertiseReceiveRefsCommand.java
index b3b5b20794..d089e008a7 100644
--- a/server-core/src/main/java/io/onedev/server/git/command/AdvertiseReceiveRefsCommand.java
+++ b/server-core/src/main/java/io/onedev/server/git/command/AdvertiseReceiveRefsCommand.java
@@ -1,56 +1,47 @@
-package io.onedev.server.git.command;
-
-import java.io.File;
-import java.io.OutputStream;
-
-import org.jspecify.annotations.Nullable;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import io.onedev.commons.utils.command.Commandline;
-import io.onedev.commons.utils.command.LineConsumer;
-import io.onedev.server.git.GitUtils;
-
-public class AdvertiseReceiveRefsCommand {
-
- private static final Logger logger = LoggerFactory.getLogger(AdvertiseReceiveRefsCommand.class);
-
- private final File workingDir;
-
- private final OutputStream output;
-
- private String protocol;
-
- public AdvertiseReceiveRefsCommand(File workingDir, OutputStream output) {
- this.workingDir = workingDir;
- this.output = output;
- }
-
- public AdvertiseReceiveRefsCommand protocol(@Nullable String protocol) {
- this.protocol = protocol;
- return this;
- }
-
- protected Commandline newGit() {
- return GitUtils.newGit();
- }
-
- public void run() {
- Commandline git = newGit().workingDir(workingDir);
-
- if (protocol != null)
- git.envs().put("GIT_PROTOCOL", protocol);
-
- git.addArgs("receive-pack", "--stateless-rpc", "--advertise-refs", ".");
- git.execute(output, new LineConsumer() {
-
- @Override
- public void consume(String line) {
- logger.error(line);
- }
-
- }).checkReturnCode();
- }
-
-}
+package io.onedev.server.git.command;
+
+import java.io.File;
+import java.io.OutputStream;
+
+import org.jspecify.annotations.Nullable;
+
+import io.onedev.commons.utils.command.Commandline;
+import io.onedev.commons.utils.command.ExecutionResult;
+import io.onedev.server.git.GitUtils;
+
+public class AdvertiseReceiveRefsCommand {
+
+ private final File workingDir;
+
+ private final OutputStream stdout;
+
+ private final OutputStream stderr;
+
+ private String protocol;
+
+ public AdvertiseReceiveRefsCommand(File workingDir, OutputStream stdout, OutputStream stderr) {
+ this.workingDir = workingDir;
+ this.stdout = stdout;
+ this.stderr = stderr;
+ }
+
+ public AdvertiseReceiveRefsCommand protocol(@Nullable String protocol) {
+ this.protocol = protocol;
+ return this;
+ }
+
+ protected Commandline newGit() {
+ return GitUtils.newGit();
+ }
+
+ public ExecutionResult run() {
+ Commandline git = newGit().workingDir(workingDir);
+
+ if (protocol != null)
+ git.envs().put("GIT_PROTOCOL", protocol);
+
+ git.addArgs("receive-pack", "--stateless-rpc", "--advertise-refs", ".");
+ return git.execute(stdout, stderr);
+ }
+
+}
diff --git a/server-core/src/main/java/io/onedev/server/git/command/AdvertiseUploadRefsCommand.java b/server-core/src/main/java/io/onedev/server/git/command/AdvertiseUploadRefsCommand.java
index ae118503ca..89a7f84182 100644
--- a/server-core/src/main/java/io/onedev/server/git/command/AdvertiseUploadRefsCommand.java
+++ b/server-core/src/main/java/io/onedev/server/git/command/AdvertiseUploadRefsCommand.java
@@ -1,54 +1,45 @@
-package io.onedev.server.git.command;
-
-import java.io.File;
-import java.io.OutputStream;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import io.onedev.commons.utils.command.Commandline;
-import io.onedev.commons.utils.command.LineConsumer;
-import io.onedev.server.git.GitUtils;
-
-public class AdvertiseUploadRefsCommand {
-
- private static final Logger logger = LoggerFactory.getLogger(AdvertiseUploadRefsCommand.class);
-
- private final File workingDir;
-
- private final OutputStream output;
-
- private String protocol;
-
- public AdvertiseUploadRefsCommand(File workingDir, OutputStream output) {
- this.workingDir = workingDir;
- this.output = output;
- }
-
- public AdvertiseUploadRefsCommand protocol(String protocol) {
- this.protocol = protocol;
- return this;
- }
-
- protected Commandline newGit() {
- return GitUtils.newGit();
- }
-
- public void run() {
- Commandline git = newGit().workingDir(workingDir);
-
- if (protocol != null)
- git.envs().put("GIT_PROTOCOL", protocol);
-
- git.addArgs("upload-pack", "--stateless-rpc", "--advertise-refs", ".");
- git.execute(output, new LineConsumer() {
-
- @Override
- public void consume(String line) {
- logger.error(line);
- }
-
- }).checkReturnCode();
- }
-
-}
+package io.onedev.server.git.command;
+
+import java.io.File;
+import java.io.OutputStream;
+
+import io.onedev.commons.utils.command.Commandline;
+import io.onedev.commons.utils.command.ExecutionResult;
+import io.onedev.server.git.GitUtils;
+
+public class AdvertiseUploadRefsCommand {
+
+ private final File workingDir;
+
+ private final OutputStream stdout;
+
+ private final OutputStream stderr;
+
+ private String protocol;
+
+ public AdvertiseUploadRefsCommand(File workingDir, OutputStream stdout, OutputStream stderr) {
+ this.workingDir = workingDir;
+ this.stdout = stdout;
+ this.stderr = stderr;
+ }
+
+ public AdvertiseUploadRefsCommand protocol(String protocol) {
+ this.protocol = protocol;
+ return this;
+ }
+
+ protected Commandline newGit() {
+ return GitUtils.newGit();
+ }
+
+ public ExecutionResult run() {
+ Commandline git = newGit().workingDir(workingDir);
+
+ if (protocol != null)
+ git.envs().put("GIT_PROTOCOL", protocol);
+
+ git.addArgs("upload-pack", "--stateless-rpc", "--advertise-refs", ".");
+ return git.execute(stdout, stderr);
+ }
+
+}
diff --git a/server-core/src/test/java/io/onedev/server/git/GitUtilsTest.java b/server-core/src/test/java/io/onedev/server/git/GitUtilsTest.java
index f4bb03b6d1..6ace3b662d 100644
--- a/server-core/src/test/java/io/onedev/server/git/GitUtilsTest.java
+++ b/server-core/src/test/java/io/onedev/server/git/GitUtilsTest.java
@@ -1,12 +1,17 @@
package io.onedev.server.git;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
@@ -18,6 +23,7 @@ import org.eclipse.jgit.lib.FileMode;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
+import org.eclipse.jgit.transport.PacketLineOut;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.util.io.NullOutputStream;
import org.junit.Test;
@@ -29,6 +35,68 @@ import io.onedev.commons.utils.FileUtils;
public class GitUtilsTest extends AbstractGitTest {
+ @Test
+ public void testAdvertiseRefsErrorResponse() {
+ var emptyDir = FileUtils.createTempDir();
+ try {
+ var stdout = new ByteArrayOutputStream();
+ GitUtils.advertiseUploadRefs(emptyDir, null, stdout, false);
+ var response = stdout.toString(StandardCharsets.UTF_8);
+ assertTrue(response.contains("ERR "));
+ assertTrue(response.contains("does not appear to be a git repository"));
+ assertTrue(response.endsWith("0000"));
+
+ stdout.reset();
+ GitUtils.advertiseReceiveRefs(emptyDir, null, stdout, false);
+ response = stdout.toString(StandardCharsets.UTF_8);
+ assertTrue(response.contains("ERR "));
+ assertTrue(response.contains("does not appear to be a git repository"));
+ assertTrue(response.endsWith("0000"));
+ } finally {
+ FileUtils.deleteDir(emptyDir);
+ }
+ }
+
+ @Test
+ public void testUploadPackErrorResponse() throws Exception {
+ var existingObject = addFileAndCommit("file", "content", "initial");
+ var missingObject = "4d70cd380dcc3d21e6450f77b8aeddd8994e1f9d";
+ var stdin = new ByteArrayOutputStream();
+ var packet = new PacketLineOut(stdin);
+ packet.writeString("command=fetch\n");
+ packet.writeString("agent=git/2.43.0\n");
+ packet.writeString("object-format=sha1\n");
+ packet.writeDelim();
+ packet.writeString("thin-pack\n");
+ packet.writeString("no-progress\n");
+ packet.writeString("ofs-delta\n");
+ packet.writeString("want " + existingObject + "\n");
+ packet.writeString("want " + missingObject + "\n");
+ packet.writeString("have " + existingObject + "\n");
+ packet.end();
+
+ var stdout = new ByteArrayOutputStream();
+ GitUtils.uploadPack(git.getRepository().getDirectory(), Collections.emptyMap(), "version=2",
+ new ByteArrayInputStream(stdin.toByteArray()), stdout, false);
+
+ var expected = new ByteArrayOutputStream();
+ packet = new PacketLineOut(expected);
+ packet.writeString("ERR upload-pack: not our ref " + missingObject);
+ assertArrayEquals(expected.toByteArray(), stdout.toByteArray());
+ }
+
+ @Test
+ public void testReceivePackErrorResponse() {
+ var stdout = new ByteArrayOutputStream();
+ GitUtils.receivePack(git.getRepository().getDirectory(), Collections.emptyMap(), null,
+ new ByteArrayInputStream("invalid request".getBytes(StandardCharsets.UTF_8)), stdout, false);
+
+ var response = stdout.toString(StandardCharsets.UTF_8);
+ assertTrue(response.contains("ERR "));
+ assertTrue(response.contains("bad line length character"));
+ assertTrue(response.endsWith("0000"));
+ }
+
@Test
public void testExactOnlyRenameDetection() {
try (var formatter = new DiffFormatter(NullOutputStream.INSTANCE)) {