The Unified and Autonomous Development Platform
Fix issue #1161 - LFS upload is very slow
Robin Shen5eaa1096113a
diff --git a/server-core/src/main/java/io/onedev/server/git/LfsObject.java b/server-core/src/main/java/io/onedev/server/git/LfsObject.javaindex c28e952e86..5188a49244 100644--- a/server-core/src/main/java/io/onedev/server/git/LfsObject.java+++ b/server-core/src/main/java/io/onedev/server/git/LfsObject.java@@ -22,6 +22,7 @@ import io.onedev.server.cluster.ClusterTask;import io.onedev.server.entitymanager.ProjectManager;import io.onedev.server.storage.StorageManager;import io.onedev.server.util.ContentDetector;+import org.jetbrains.annotations.NotNull;public class LfsObject implements Serializable {@@ -115,6 +116,10 @@ public class LfsObject implements Serializable {writeLock.lock();try {return new FilterOutputStream(new FileOutputStream(getFile())) {+ @Override+ public void write(@NotNull byte[] b, int off, int len) throws IOException {+ out.write(b, off, len);+ }@Overridepublic void close() throws IOException {diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/blob/ProjectBlobPage.html b/server-core/src/main/java/io/onedev/server/web/page/project/blob/ProjectBlobPage.htmlindex bc18c394db..da6a299c48 100644--- a/server-core/src/main/java/io/onedev/server/web/page/project/blob/ProjectBlobPage.html+++ b/server-core/src/main/java/io/onedev/server/web/page/project/blob/ProjectBlobPage.html@@ -29,7 +29,12 @@</div><wicket:fragment wicket:id="binaryOrLargeFileFrag"><div class="border-bottom d-flex align-items-center px-3 py-2 justify-content-between">+ <span class="text-gray"><span wicket:id="size"></span>+ <wicket:enclosure child="lfsHint">+ <span wicket:id="lfsHint" class="text-muted">|</span> Stored with Git LFS+ </wicket:enclosure>+ </span><span><a wicket:id="download" class="btn btn-icon btn-xs btn-light btn-hover-primary ml-4" title="Download"><wicket:svg href="download" class="icon"/></a><span wicket:id="delete" class="ml-1">diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/blob/ProjectBlobPage.java b/server-core/src/main/java/io/onedev/server/web/page/project/blob/ProjectBlobPage.javaindex 8a854a8352..428302d54e 100644--- a/server-core/src/main/java/io/onedev/server/web/page/project/blob/ProjectBlobPage.java+++ b/server-core/src/main/java/io/onedev/server/web/page/project/blob/ProjectBlobPage.java@@ -1,61 +1,9 @@package io.onedev.server.web.page.project.blob;-import static org.apache.wicket.ajax.attributes.CallbackParameter.explicit;--import java.io.Serializable;-import java.net.URISyntaxException;-import java.util.ArrayList;-import java.util.Collection;-import java.util.HashMap;-import java.util.HashSet;-import java.util.List;-import java.util.Map;-import java.util.Set;-import java.util.concurrent.atomic.AtomicBoolean;--import javax.annotation.Nullable;--import org.apache.commons.lang3.SerializationUtils;-import org.apache.commons.lang3.StringUtils;-import org.apache.http.NameValuePair;-import org.apache.http.client.utils.URIBuilder;-import org.apache.wicket.Component;-import org.apache.wicket.Session;-import org.apache.wicket.ajax.AjaxRequestTarget;-import org.apache.wicket.ajax.attributes.AjaxRequestAttributes;-import org.apache.wicket.ajax.markup.html.AjaxLink;-import org.apache.wicket.behavior.AttributeAppender;-import org.apache.wicket.core.request.handler.IPartialPageRequestHandler;-import org.apache.wicket.event.IEvent;-import org.apache.wicket.markup.ComponentTag;-import org.apache.wicket.markup.head.IHeaderResponse;-import org.apache.wicket.markup.head.JavaScriptHeaderItem;-import org.apache.wicket.markup.head.OnDomReadyHeaderItem;-import org.apache.wicket.markup.html.WebMarkupContainer;-import org.apache.wicket.markup.html.basic.Label;-import org.apache.wicket.markup.html.link.ResourceLink;-import org.apache.wicket.markup.html.panel.Fragment;-import org.apache.wicket.model.AbstractReadOnlyModel;-import org.apache.wicket.model.IModel;-import org.apache.wicket.model.Model;-import org.apache.wicket.request.IRequestParameters;-import org.apache.wicket.request.cycle.RequestCycle;-import org.apache.wicket.request.flow.RedirectToUrlException;-import org.apache.wicket.request.handler.resource.ResourceReferenceRequestHandler;-import org.apache.wicket.request.mapper.parameter.PageParameters;-import org.apache.wicket.util.visit.IVisit;-import org.apache.wicket.util.visit.IVisitor;-import org.eclipse.jgit.lib.FileMode;-import org.eclipse.jgit.lib.ObjectId;-import org.eclipse.jgit.revwalk.RevCommit;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;-import com.google.common.base.Objects;import com.google.common.base.Preconditions;import com.google.common.base.Splitter;import com.google.common.collect.Sets;-import io.onedev.commons.utils.ExceptionUtils;import io.onedev.commons.utils.FileUtils;import io.onedev.commons.utils.PlanarRange;@@ -65,12 +13,7 @@ import io.onedev.server.entitymanager.CodeCommentManager;import io.onedev.server.entitymanager.PullRequestManager;import io.onedev.server.entitymanager.SettingManager;import io.onedev.server.event.project.CommitIndexed;-import io.onedev.server.git.BlobContent;-import io.onedev.server.git.BlobEdits;-import io.onedev.server.git.BlobIdent;-import io.onedev.server.git.GitUtils;-import io.onedev.server.git.LfsObject;-import io.onedev.server.git.LfsPointer;+import io.onedev.server.git.*;import io.onedev.server.git.exception.BlobEditException;import io.onedev.server.git.exception.NotTreeException;import io.onedev.server.git.exception.ObjectAlreadyExistsException;@@ -127,6 +70,49 @@ import io.onedev.server.web.resource.RawBlobResourceReference;import io.onedev.server.web.util.EditParamsAware;import io.onedev.server.web.util.FileUpload;import io.onedev.server.web.websocket.WebSocketManager;+import org.apache.commons.lang3.SerializationUtils;+import org.apache.commons.lang3.StringUtils;+import org.apache.http.NameValuePair;+import org.apache.http.client.utils.URIBuilder;+import org.apache.wicket.Component;+import org.apache.wicket.Session;+import org.apache.wicket.ajax.AjaxRequestTarget;+import org.apache.wicket.ajax.attributes.AjaxRequestAttributes;+import org.apache.wicket.ajax.markup.html.AjaxLink;+import org.apache.wicket.behavior.AttributeAppender;+import org.apache.wicket.core.request.handler.IPartialPageRequestHandler;+import org.apache.wicket.event.IEvent;+import org.apache.wicket.markup.ComponentTag;+import org.apache.wicket.markup.head.IHeaderResponse;+import org.apache.wicket.markup.head.JavaScriptHeaderItem;+import org.apache.wicket.markup.head.OnDomReadyHeaderItem;+import org.apache.wicket.markup.html.WebMarkupContainer;+import org.apache.wicket.markup.html.basic.Label;+import org.apache.wicket.markup.html.link.ResourceLink;+import org.apache.wicket.markup.html.panel.Fragment;+import org.apache.wicket.model.AbstractReadOnlyModel;+import org.apache.wicket.model.IModel;+import org.apache.wicket.model.Model;+import org.apache.wicket.request.IRequestParameters;+import org.apache.wicket.request.cycle.RequestCycle;+import org.apache.wicket.request.flow.RedirectToUrlException;+import org.apache.wicket.request.handler.resource.ResourceReferenceRequestHandler;+import org.apache.wicket.request.mapper.parameter.PageParameters;+import org.apache.wicket.util.visit.IVisit;+import org.apache.wicket.util.visit.IVisitor;+import org.eclipse.jgit.lib.FileMode;+import org.eclipse.jgit.lib.ObjectId;+import org.eclipse.jgit.revwalk.RevCommit;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++import javax.annotation.Nullable;+import java.io.Serializable;+import java.net.URISyntaxException;+import java.util.*;+import java.util.concurrent.atomic.AtomicBoolean;++import static org.apache.wicket.ajax.attributes.CallbackParameter.explicit;@SuppressWarnings("serial")public class ProjectBlobPage extends ProjectPage implements BlobRenderContext,@@ -737,6 +723,10 @@ public class ProjectBlobPage extends ProjectPage implements BlobRenderContext,return advancedSearchPanel;}+ private Blob getBlob(boolean mustExist) {+ return getProject().getBlob(getBlobIdent(), mustExist);+ }+private void newBlobContent(@Nullable AjaxRequestTarget target) {Component blobContent = null;if (getMode() == Mode.VIEW && getBlobIdent().revision == null) {@@ -749,7 +739,7 @@ public class ProjectBlobPage extends ProjectPage implements BlobRenderContext,blobContent = new NoNameEditPanel(BLOB_CONTENT_ID, this);} else {if (getMode() == Mode.VIEW) {- LfsPointer lfsPointer = getProject().getBlob(getBlobIdent(), true).getLfsPointer();+ LfsPointer lfsPointer = getBlob(true).getLfsPointer();if (lfsPointer != null && !new LfsObject(getProject().getId(), lfsPointer.getObjectId()).exists())blobContent = new Fragment(BLOB_CONTENT_ID, "lfsObjectMissingFrag", this);}@@ -765,27 +755,32 @@ public class ProjectBlobPage extends ProjectPage implements BlobRenderContext,if (blobContent == null) {if (getMode() == Mode.ADD|| getMode() == Mode.EDIT- && getProject().getBlob(getBlobIdent(), true).getText() != null- && getProject().getBlob(getBlobIdent(), true).getLfsPointer() == null- && !getProject().getBlob(getBlobIdent(), true).isPartial()) {+ && getBlob(true).getText() != null+ && getBlob(true).getLfsPointer() == null+ && !getBlob(true).isPartial()) {blobContent = new SourceEditPanel(BLOB_CONTENT_ID, this);} else if ((getMode() == Mode.VIEW || getMode() == Mode.BLAME)&& getBlobIdent().isFile()- && getProject().getBlob(getBlobIdent(), true).getText() != null- && getProject().getBlob(getBlobIdent(), true).getLfsPointer() == null) {+ && getBlob(true).getText() != null+ && getBlob(true).getLfsPointer() == null) {blobContent = new SourceViewPanel(BLOB_CONTENT_ID, this, false);} else {Fragment fragment = new Fragment(BLOB_CONTENT_ID, "binaryOrLargeFileFrag", this);-- if (getProject().getBlob(getBlobIdent(), true).getText() == null- || getProject().getBlob(getBlobIdent(), true).getLfsPointer() != null) {++ Blob blob = getBlob(true);+ if (blob.getText() == null || blob.getLfsPointer() != null) {fragment.add(new Label("message", "Binary file"));} else {fragment.add(new Label("message", "File is too large"));}-- long size = getProject().getBlob(getBlobIdent(), true).getSize();++ long size;+ if (blob.getLfsPointer() != null)+ size = blob.getLfsPointer().getObjectSize();+ else+ size = blob.getSize();fragment.add(new Label("size", FileUtils.byteCountToDisplaySize(size)));+ fragment.add(new WebMarkupContainer("lfsHint").setVisible(blob.getLfsPointer() != null));fragment.add(new ResourceLink<Void>("download", new RawBlobResourceReference(),RawBlobResource.paramsOf(getProject(), getBlobIdent())));@@ -936,7 +931,8 @@ public class ProjectBlobPage extends ProjectPage implements BlobRenderContext,if (resolvedRevision != null && isOnBranch() && state.blobIdent.path == null && state.mode == Mode.VIEW) {BlobIdent oldBlobIdent = new BlobIdent(resolvedRevision.name(), ".onedev-buildspec", FileMode.TYPE_FILE);BlobIdent blobIdent = new BlobIdent(resolvedRevision.name(), BuildSpec.BLOB_PATH, FileMode.TYPE_FILE);- setVisible(getProject().getBlob(blobIdent, false) == null && getProject().getBlob(oldBlobIdent, false) == null);+ setVisible(getProject().getBlob(blobIdent, false) == null+ && getProject().getBlob(oldBlobIdent, false) == null);} else {setVisible(false);}