The Unified and Autonomous Development Platform
refactor: Better error handling for uploadpack/receivepack/advertiseuploadrefs/advertisereceiverefs
Robin Shen76d17420452a
diff --git a/server-core/src/main/java/io/onedev/server/cluster/ClusterResource.java b/server-core/src/main/java/io/onedev/server/cluster/ClusterResource.javaindex 4d58443584..ad687dd34e 100644--- a/server-core/src/main/java/io/onedev/server/cluster/ClusterResource.java+++ b/server-core/src/main/java/io/onedev/server/cluster/ClusterResource.java@@ -46,8 +46,6 @@ import io.onedev.server.codequality.UnitTestReport;import io.onedev.server.git.GitFilter;import io.onedev.server.git.GitUtils;import io.onedev.server.git.LfsObject;-import io.onedev.server.git.command.AdvertiseReceiveRefsCommand;-import io.onedev.server.git.command.AdvertiseUploadRefsCommand;import io.onedev.server.git.hook.HookUtils;import io.onedev.server.model.PackBlob;import io.onedev.server.model.Project;@@ -275,9 +273,9 @@ public class ClusterResource {StreamingOutput os = output -> {File gitDir = projectService.getGitDir(projectId);if (upload)- new AdvertiseUploadRefsCommand(gitDir, output).protocol(protocol).run();+ GitUtils.advertiseUploadRefs(gitDir, protocol, output);else- new AdvertiseReceiveRefsCommand(gitDir, output).protocol(protocol).run();+ GitUtils.advertiseReceiveRefs(gitDir, protocol, output);};return ok(os).build();}diff --git a/server-core/src/main/java/io/onedev/server/git/GitFilter.java b/server-core/src/main/java/io/onedev/server/git/GitFilter.javaindex 22499ecc64..0be34f586d 100644--- a/server-core/src/main/java/io/onedev/server/git/GitFilter.java+++ b/server-core/src/main/java/io/onedev/server/git/GitFilter.java@@ -45,8 +45,6 @@ import io.onedev.k8shelper.KubernetesHelper;import io.onedev.server.OneDev;import io.onedev.server.cluster.ClusterService;import io.onedev.server.exception.ServerNotReadyException;-import io.onedev.server.git.command.AdvertiseReceiveRefsCommand;-import io.onedev.server.git.command.AdvertiseUploadRefsCommand;import io.onedev.server.git.hook.HookUtils;import io.onedev.server.model.Project;import io.onedev.server.persistence.SessionService;@@ -337,10 +335,10 @@ public class GitFilter implements Filter {String activeServer = projectService.getActiveServer(projectId, true);if (activeServer.equals(clusterService.getLocalServerAddress())) {File gitDir = projectService.getGitDir(projectId);- if (upload)- new AdvertiseUploadRefsCommand(gitDir, output).protocol(protocol).run();- else- new AdvertiseReceiveRefsCommand(gitDir, output).protocol(protocol).run();+ if (upload)+ GitUtils.advertiseUploadRefs(gitDir, protocol, output);+ else+ GitUtils.advertiseReceiveRefs(gitDir, protocol, output);} else {Client client = ClientBuilder.newClient();try {diff --git a/server-core/src/main/java/io/onedev/server/git/GitUtils.java b/server-core/src/main/java/io/onedev/server/git/GitUtils.javaindex 76aaa126aa..1619544344 100644--- a/server-core/src/main/java/io/onedev/server/git/GitUtils.java+++ b/server-core/src/main/java/io/onedev/server/git/GitUtils.java@@ -19,6 +19,7 @@ import java.util.List;import java.util.Map;import java.util.StringTokenizer;import java.util.concurrent.atomic.AtomicBoolean;+import java.util.concurrent.atomic.AtomicReference;import java.util.stream.Collectors;import org.apache.commons.lang3.Strings;@@ -63,6 +64,7 @@ import org.eclipse.jgit.revwalk.RevTree;import org.eclipse.jgit.revwalk.RevWalk;import org.eclipse.jgit.revwalk.RevWalkUtils;import org.eclipse.jgit.revwalk.filter.RevFilter;+import org.eclipse.jgit.transport.PacketLineOut;import org.eclipse.jgit.treewalk.CanonicalTreeParser;import org.eclipse.jgit.treewalk.TreeWalk;import org.eclipse.jgit.treewalk.filter.TreeFilter;@@ -93,6 +95,8 @@ import io.onedev.k8shelper.KubernetesHelper;import io.onedev.server.OneDev;import io.onedev.server.cluster.ClusterService;import io.onedev.server.exception.NotFoundException;+import io.onedev.server.git.command.AdvertiseReceiveRefsCommand;+import io.onedev.server.git.command.AdvertiseUploadRefsCommand;import io.onedev.server.git.command.FileChange;import io.onedev.server.git.command.IsAncestorCommand;import io.onedev.server.git.command.ReceivePackCommand;@@ -857,9 +861,44 @@ public class GitUtils {return change;}- public static void uploadPack(File gitDir, Map<String, String> environments, String protocol,+ public static void advertiseUploadRefs(File gitDir, String protocol, OutputStream stdout) {+ advertiseUploadRefs(gitDir, protocol, stdout, true);+ }++ public static void advertiseUploadRefs(File gitDir, String protocol, OutputStream stdout, boolean logError) {+ AtomicBoolean outputWritten = new AtomicBoolean(false);+ var errorMessage = new AtomicReference<String>();+ var advertise = new AdvertiseUploadRefsCommand(gitDir, trackOutput(stdout, outputWritten),+ errorLogger(errorMessage, logError));+ var result = advertise.protocol(protocol).run();+ handleGitResult(result, stdout, outputWritten, errorMessage.get(),+ "git upload-pack --advertise-refs failed", logError);+ }++ public static void advertiseReceiveRefs(File gitDir, String protocol, OutputStream stdout) {+ advertiseReceiveRefs(gitDir, protocol, stdout, true);+ }++ public static void advertiseReceiveRefs(File gitDir, String protocol, OutputStream stdout, boolean logError) {+ AtomicBoolean outputWritten = new AtomicBoolean(false);+ var errorMessage = new AtomicReference<String>();+ var advertise = new AdvertiseReceiveRefsCommand(gitDir, trackOutput(stdout, outputWritten),+ errorLogger(errorMessage, logError));+ var result = advertise.protocol(protocol).run();+ handleGitResult(result, stdout, outputWritten, errorMessage.get(),+ "git receive-pack --advertise-refs failed", logError);+ }++ public static void uploadPack(File gitDir, Map<String, String> environments, String protocol,InputStream stdin, OutputStream stdout) {+ uploadPack(gitDir, environments, protocol, stdin, stdout, true);+ }++ public static void uploadPack(File gitDir, Map<String, String> environments, String protocol,+ InputStream stdin, OutputStream stdout, boolean logError) {AtomicBoolean toleratedErrors = new AtomicBoolean(false);+ AtomicBoolean outputWritten = new AtomicBoolean(false);+ var errorMessage = new AtomicReference<String>();var stderr = new LineConsumer(UTF_8.name()) {@Override@@ -869,35 +908,93 @@ public class GitUtils {toleratedErrors.set(true);logger.debug(line);} else {- logger.error(line);+ errorMessage.compareAndSet(null, line);+ if (logError)+ logger.error(line);}}};- ExecutionResult result;- var upload = new UploadPackCommand(gitDir, stdin, stdout, stderr, environments);+ var upload = new UploadPackCommand(gitDir, stdin, trackOutput(stdout, outputWritten), stderr, environments);upload.statelessRpc(true).protocol(protocol);- result = upload.run();- if (result.getReturnCode() != 0 && !toleratedErrors.get())- throw result.buildException();+ var result = upload.run();+ if (!toleratedErrors.get())+ handleGitResult(result, stdout, outputWritten, errorMessage.get(), "git upload-pack failed", logError);}-+public static void receivePack(File gitDir, Map<String, String> environments, String protocol,InputStream stdin, OutputStream stdout) {- var stderr = new LineConsumer(UTF_8.name()) {+ receivePack(gitDir, environments, protocol, stdin, stdout, true);+ }++ public static void receivePack(File gitDir, Map<String, String> environments, String protocol,+ InputStream stdin, OutputStream stdout, boolean logError) {+ AtomicBoolean outputWritten = new AtomicBoolean(false);+ var errorMessage = new AtomicReference<String>();+ var receive = new ReceivePackCommand(gitDir, stdin, trackOutput(stdout, outputWritten),+ errorLogger(errorMessage, logError), environments);+ receive.statelessRpc(true).protocol(protocol);+ ExecutionResult result = receive.run();+ handleGitResult(result, stdout, outputWritten, errorMessage.get(), "git receive-pack failed", logError);+ }++ private static LineConsumer errorLogger(AtomicReference<String> errorMessage, boolean logError) {+ return new LineConsumer(UTF_8.name()) {@Overridepublic void consume(String line) {- logger.error(line);+ errorMessage.compareAndSet(null, line);+ if (logError)+ logger.error(line);}-+};-- var receive = new ReceivePackCommand(gitDir, stdin, stdout, stderr, environments);- receive.statelessRpc(true).protocol(protocol);- ExecutionResult result = receive.run();- result.checkReturnCode();+ }++ private static OutputStream trackOutput(OutputStream output, AtomicBoolean outputWritten) {+ return new OutputStream() {++ @Override+ public void write(int b) throws IOException {+ output.write(b);+ outputWritten.set(true);+ }++ @Override+ public void write(byte[] b, int off, int len) throws IOException {+ output.write(b, off, len);+ if (len != 0)+ outputWritten.set(true);+ }++ @Override+ public void flush() throws IOException {+ output.flush();+ }++ };+ }++ private static void handleGitResult(ExecutionResult result, OutputStream output, AtomicBoolean outputWritten,+ String errorMessage, String fallbackMessage, boolean logError) {+ if (result.getReturnCode() != 0) {+ if (!outputWritten.get()) {+ var message = errorMessage;+ if (message == null)+ message = fallbackMessage;+ try {+ var packet = new PacketLineOut(output);+ packet.writeString("ERR " + message + "\n");+ packet.end();+ output.flush();+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ } else if (errorMessage == null && logError) {+ logger.error("{} (exit code: {})", fallbackMessage, result.getReturnCode());+ }+ }}}diff --git a/server-core/src/main/java/io/onedev/server/git/command/AdvertiseReceiveRefsCommand.java b/server-core/src/main/java/io/onedev/server/git/command/AdvertiseReceiveRefsCommand.javaindex b3b5b20794..d089e008a7 100644--- a/server-core/src/main/java/io/onedev/server/git/command/AdvertiseReceiveRefsCommand.java+++ b/server-core/src/main/java/io/onedev/server/git/command/AdvertiseReceiveRefsCommand.java@@ -1,56 +1,47 @@-package io.onedev.server.git.command;--import java.io.File;-import java.io.OutputStream;--import org.jspecify.annotations.Nullable;--import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--import io.onedev.commons.utils.command.Commandline;-import io.onedev.commons.utils.command.LineConsumer;-import io.onedev.server.git.GitUtils;--public class AdvertiseReceiveRefsCommand {-- private static final Logger logger = LoggerFactory.getLogger(AdvertiseReceiveRefsCommand.class);-- private final File workingDir;-- private final OutputStream output;-- private String protocol;-- public AdvertiseReceiveRefsCommand(File workingDir, OutputStream output) {- this.workingDir = workingDir;- this.output = output;- }-- public AdvertiseReceiveRefsCommand protocol(@Nullable String protocol) {- this.protocol = protocol;- return this;- }-- protected Commandline newGit() {- return GitUtils.newGit();- }-- public void run() {- Commandline git = newGit().workingDir(workingDir);-- if (protocol != null)- git.envs().put("GIT_PROTOCOL", protocol);-- git.addArgs("receive-pack", "--stateless-rpc", "--advertise-refs", ".");- git.execute(output, new LineConsumer() {-- @Override- public void consume(String line) {- logger.error(line);- }-- }).checkReturnCode();- }--}+package io.onedev.server.git.command;++import java.io.File;+import java.io.OutputStream;++import org.jspecify.annotations.Nullable;++import io.onedev.commons.utils.command.Commandline;+import io.onedev.commons.utils.command.ExecutionResult;+import io.onedev.server.git.GitUtils;++public class AdvertiseReceiveRefsCommand {++ private final File workingDir;++ private final OutputStream stdout;++ private final OutputStream stderr;++ private String protocol;++ public AdvertiseReceiveRefsCommand(File workingDir, OutputStream stdout, OutputStream stderr) {+ this.workingDir = workingDir;+ this.stdout = stdout;+ this.stderr = stderr;+ }++ public AdvertiseReceiveRefsCommand protocol(@Nullable String protocol) {+ this.protocol = protocol;+ return this;+ }++ protected Commandline newGit() {+ return GitUtils.newGit();+ }++ public ExecutionResult run() {+ Commandline git = newGit().workingDir(workingDir);++ if (protocol != null)+ git.envs().put("GIT_PROTOCOL", protocol);++ git.addArgs("receive-pack", "--stateless-rpc", "--advertise-refs", ".");+ return git.execute(stdout, stderr);+ }++}diff --git a/server-core/src/main/java/io/onedev/server/git/command/AdvertiseUploadRefsCommand.java b/server-core/src/main/java/io/onedev/server/git/command/AdvertiseUploadRefsCommand.javaindex ae118503ca..89a7f84182 100644--- a/server-core/src/main/java/io/onedev/server/git/command/AdvertiseUploadRefsCommand.java+++ b/server-core/src/main/java/io/onedev/server/git/command/AdvertiseUploadRefsCommand.java@@ -1,54 +1,45 @@-package io.onedev.server.git.command;--import java.io.File;-import java.io.OutputStream;--import org.slf4j.Logger;-import org.slf4j.LoggerFactory;--import io.onedev.commons.utils.command.Commandline;-import io.onedev.commons.utils.command.LineConsumer;-import io.onedev.server.git.GitUtils;--public class AdvertiseUploadRefsCommand {-- private static final Logger logger = LoggerFactory.getLogger(AdvertiseUploadRefsCommand.class);-- private final File workingDir;-- private final OutputStream output;-- private String protocol;-- public AdvertiseUploadRefsCommand(File workingDir, OutputStream output) {- this.workingDir = workingDir;- this.output = output;- }-- public AdvertiseUploadRefsCommand protocol(String protocol) {- this.protocol = protocol;- return this;- }-- protected Commandline newGit() {- return GitUtils.newGit();- }-- public void run() {- Commandline git = newGit().workingDir(workingDir);-- if (protocol != null)- git.envs().put("GIT_PROTOCOL", protocol);-- git.addArgs("upload-pack", "--stateless-rpc", "--advertise-refs", ".");- git.execute(output, new LineConsumer() {-- @Override- public void consume(String line) {- logger.error(line);- }-- }).checkReturnCode();- }--}+package io.onedev.server.git.command;++import java.io.File;+import java.io.OutputStream;++import io.onedev.commons.utils.command.Commandline;+import io.onedev.commons.utils.command.ExecutionResult;+import io.onedev.server.git.GitUtils;++public class AdvertiseUploadRefsCommand {++ private final File workingDir;++ private final OutputStream stdout;++ private final OutputStream stderr;++ private String protocol;++ public AdvertiseUploadRefsCommand(File workingDir, OutputStream stdout, OutputStream stderr) {+ this.workingDir = workingDir;+ this.stdout = stdout;+ this.stderr = stderr;+ }++ public AdvertiseUploadRefsCommand protocol(String protocol) {+ this.protocol = protocol;+ return this;+ }++ protected Commandline newGit() {+ return GitUtils.newGit();+ }++ public ExecutionResult run() {+ Commandline git = newGit().workingDir(workingDir);++ if (protocol != null)+ git.envs().put("GIT_PROTOCOL", protocol);++ git.addArgs("upload-pack", "--stateless-rpc", "--advertise-refs", ".");+ return git.execute(stdout, stderr);+ }++}diff --git a/server-core/src/test/java/io/onedev/server/git/GitUtilsTest.java b/server-core/src/test/java/io/onedev/server/git/GitUtilsTest.javaindex f4bb03b6d1..6ace3b662d 100644--- a/server-core/src/test/java/io/onedev/server/git/GitUtilsTest.java+++ b/server-core/src/test/java/io/onedev/server/git/GitUtilsTest.java@@ -1,12 +1,17 @@package io.onedev.server.git;+import static org.junit.Assert.assertArrayEquals;import static org.junit.Assert.assertEquals;import static org.junit.Assert.assertNotNull;import static org.junit.Assert.assertNull;import static org.junit.Assert.assertTrue;+import java.io.ByteArrayInputStream;+import java.io.ByteArrayOutputStream;import java.io.File;import java.io.InputStream;+import java.nio.charset.StandardCharsets;+import java.util.Collections;import java.util.HashSet;import java.util.Set;@@ -18,6 +23,7 @@ import org.eclipse.jgit.lib.FileMode;import org.eclipse.jgit.lib.ObjectId;import org.eclipse.jgit.revwalk.RevCommit;import org.eclipse.jgit.revwalk.RevWalk;+import org.eclipse.jgit.transport.PacketLineOut;import org.eclipse.jgit.treewalk.TreeWalk;import org.eclipse.jgit.util.io.NullOutputStream;import org.junit.Test;@@ -29,6 +35,68 @@ import io.onedev.commons.utils.FileUtils;public class GitUtilsTest extends AbstractGitTest {+ @Test+ public void testAdvertiseRefsErrorResponse() {+ var emptyDir = FileUtils.createTempDir();+ try {+ var stdout = new ByteArrayOutputStream();+ GitUtils.advertiseUploadRefs(emptyDir, null, stdout, false);+ var response = stdout.toString(StandardCharsets.UTF_8);+ assertTrue(response.contains("ERR "));+ assertTrue(response.contains("does not appear to be a git repository"));+ assertTrue(response.endsWith("0000"));++ stdout.reset();+ GitUtils.advertiseReceiveRefs(emptyDir, null, stdout, false);+ response = stdout.toString(StandardCharsets.UTF_8);+ assertTrue(response.contains("ERR "));+ assertTrue(response.contains("does not appear to be a git repository"));+ assertTrue(response.endsWith("0000"));+ } finally {+ FileUtils.deleteDir(emptyDir);+ }+ }++ @Test+ public void testUploadPackErrorResponse() throws Exception {+ var existingObject = addFileAndCommit("file", "content", "initial");+ var missingObject = "4d70cd380dcc3d21e6450f77b8aeddd8994e1f9d";+ var stdin = new ByteArrayOutputStream();+ var packet = new PacketLineOut(stdin);+ packet.writeString("command=fetch\n");+ packet.writeString("agent=git/2.43.0\n");+ packet.writeString("object-format=sha1\n");+ packet.writeDelim();+ packet.writeString("thin-pack\n");+ packet.writeString("no-progress\n");+ packet.writeString("ofs-delta\n");+ packet.writeString("want " + existingObject + "\n");+ packet.writeString("want " + missingObject + "\n");+ packet.writeString("have " + existingObject + "\n");+ packet.end();++ var stdout = new ByteArrayOutputStream();+ GitUtils.uploadPack(git.getRepository().getDirectory(), Collections.emptyMap(), "version=2",+ new ByteArrayInputStream(stdin.toByteArray()), stdout, false);++ var expected = new ByteArrayOutputStream();+ packet = new PacketLineOut(expected);+ packet.writeString("ERR upload-pack: not our ref " + missingObject);+ assertArrayEquals(expected.toByteArray(), stdout.toByteArray());+ }++ @Test+ public void testReceivePackErrorResponse() {+ var stdout = new ByteArrayOutputStream();+ GitUtils.receivePack(git.getRepository().getDirectory(), Collections.emptyMap(), null,+ new ByteArrayInputStream("invalid request".getBytes(StandardCharsets.UTF_8)), stdout, false);++ var response = stdout.toString(StandardCharsets.UTF_8);+ assertTrue(response.contains("ERR "));+ assertTrue(response.contains("bad line length character"));+ assertTrue(response.endsWith("0000"));+ }+@Testpublic void testExactOnlyRenameDetection() {try (var formatter = new DiffFormatter(NullOutputStream.INSTANCE)) {