The Unified and Autonomous Development Platform
chore: Build log streaming restful api
Robin Shenec0db843eafb
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/job/log/JobLogEntryEx.java b/server-core/src/main/java/io/onedev/server/buildspec/job/log/JobLogEntryEx.javaindex 7520366bea..aff9b8c37e 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/job/log/JobLogEntryEx.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/job/log/JobLogEntryEx.java@@ -14,6 +14,7 @@ import com.google.common.base.Splitter;import com.google.common.collect.Lists;import io.onedev.server.job.log.StyleBuilder;+import io.onedev.server.web.asset.emoji.Emojis;public class JobLogEntryEx implements Serializable {@@ -32,6 +33,14 @@ public class JobLogEntryEx implements Serializable {this(entry.getDate(), Lists.newArrayList(new Message(new StyleBuilder().build(), entry.getMessage())));}+ public JobLogEntryEx transformEmojis() {+ Emojis emojis = Emojis.getInstance();+ List<Message> messages = new ArrayList<>();+ for (Message message: getMessages())+ messages.add(new Message(message.getStyle(), emojis.apply(message.getText())));+ return new JobLogEntryEx(getDate(), messages);+ }+// Handle ANSI escape codes according to https://en.wikipedia.org/wiki/ANSI_escape_codepublic static JobLogEntryEx parse(String text, StyleBuilder styleBuilder) {text = text.replace("\r\n", "\n");diff --git a/server-core/src/main/java/io/onedev/server/job/log/DefaultLogManager.java b/server-core/src/main/java/io/onedev/server/job/log/DefaultLogManager.javaindex 4a2b5105b6..6f5aae5e31 100644--- a/server-core/src/main/java/io/onedev/server/job/log/DefaultLogManager.java+++ b/server-core/src/main/java/io/onedev/server/job/log/DefaultLogManager.java@@ -11,12 +11,14 @@ import io.onedev.server.buildspec.job.log.instruction.LogInstruction;import io.onedev.server.buildspec.job.log.instruction.LogInstructionParser.InstructionContext;import io.onedev.server.buildspec.job.log.instruction.LogInstructionParser.ParamContext;import io.onedev.server.buildspecmodel.inputspec.SecretInput;+import io.onedev.server.cluster.ClusterManager;import io.onedev.server.cluster.ClusterTask;import io.onedev.server.entitymanager.BuildManager;import io.onedev.server.entitymanager.ProjectManager;import io.onedev.server.event.Listen;import io.onedev.server.event.project.build.BuildFinished;import io.onedev.server.model.Build;+import io.onedev.server.persistence.TransactionManager;import io.onedev.server.persistence.annotation.Sessional;import io.onedev.server.web.websocket.WebSocketManager;import org.antlr.v4.runtime.tree.TerminalNode;@@ -34,6 +36,8 @@ import java.nio.charset.StandardCharsets;import java.util.*;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.locks.Lock;+import java.util.concurrent.locks.ReadWriteLock;+import java.util.concurrent.locks.ReentrantReadWriteLock;import java.util.regex.Pattern;import static io.onedev.commons.utils.LockUtils.*;@@ -58,15 +62,27 @@ public class DefaultLogManager implements LogManager, Serializable {private final BuildManager buildManager;+ private final ClusterManager clusterManager;++ private final TransactionManager transactionManager;+private final Map<String, LogSnippet> recentSnippets = new ConcurrentHashMap<>();private final Map<String, TaskLogger> jobLoggers = new ConcurrentHashMap<>();+ private final ReadWriteLock logListenersLock = new ReentrantReadWriteLock();++ private final List<LogListener> logListeners = new ArrayList<>();+@Inject- public DefaultLogManager(WebSocketManager webSocketManager, ProjectManager projectManager, BuildManager buildManager) {+ public DefaultLogManager(WebSocketManager webSocketManager, ProjectManager projectManager,+ BuildManager buildManager, ClusterManager clusterManager,+ TransactionManager transactionManager) {this.projectManager = projectManager;this.webSocketManager = webSocketManager;this.buildManager = buildManager;+ this.clusterManager = clusterManager;+ this.transactionManager = transactionManager;}public Object writeReplace() throws ObjectStreamException {@@ -78,6 +94,20 @@ public class DefaultLogManager implements LogManager, Serializable {return newLogger(build, new ArrayList<>());}+ private void notifyListeners(Long buildId) {+ clusterManager.submitToAllServers(() -> {+ var lock = logListenersLock.readLock();+ lock.lock();+ try {+ for (var logListener: logListeners)+ logListener.logged(buildId);+ } finally {+ lock.unlock();+ }+ return null;+ });+ }+@Overridepublic TaskLogger newLogger(Build build, Collection<String> jobSecretsToMask) {Long projectId = build.getProject().getId();@@ -125,6 +155,7 @@ public class DefaultLogManager implements LogManager, Serializable {}}webSocketManager.notifyObservableChange(Build.getLogChangeObservable(buildId), null);+ notifyListeners(buildId);}}return null;@@ -291,11 +322,8 @@ public class DefaultLogManager implements LogManager, Serializable {}}- @Sessional@Override- public List<JobLogEntryEx> readLogEntries(Build build, int from, int count) {- Long projectId = build.getProject().getId();- Long buildNumber = build.getNumber();+ public List<JobLogEntryEx> readLogEntries(Long projectId, Long buildNumber, int from, int count) {return projectManager.runOnActiveServer(projectId, new ClusterTask<>() {private static final long serialVersionUID = 1L;@@ -325,11 +353,8 @@ public class DefaultLogManager implements LogManager, Serializable {});}- @Sessional@Override- public LogSnippet readLogSnippetReversely(Build build, int count) {- Long projectId = build.getProject().getId();- Long buildNumber = build.getNumber();+ public LogSnippet readLogSnippetReversely(Long projectId, Long buildNumber, int count) {return projectManager.runOnActiveServer(projectId, new ClusterTask<>() {private static final long serialVersionUID = 1L;@@ -397,6 +422,7 @@ public class DefaultLogManager implements LogManager, Serializable {}return null;});+ transactionManager.runAfterCommit(() -> notifyListeners(build.getId()));}private String getLogKey(Long projectId, Long buildNumber) {@@ -501,6 +527,28 @@ public class DefaultLogManager implements LogManager, Serializable {}+ @Override+ public void registerListener(LogListener listener) {+ var lock = logListenersLock.writeLock();+ lock.lock();+ try {+ logListeners.add(listener);+ } finally {+ lock.unlock();+ }+ }++ @Override+ public void deregisterListener(LogListener listener) {+ var lock = logListenersLock.writeLock();+ lock.lock();+ try {+ logListeners.remove(listener);+ } finally {+ lock.unlock();+ }+ }+@Overridepublic TaskLogger getJobLogger(String jobToken) {return jobLoggers.get(jobToken);diff --git a/server-core/src/main/java/io/onedev/server/job/log/LogListener.java b/server-core/src/main/java/io/onedev/server/job/log/LogListener.javanew file mode 100644index 0000000000..5f63dbbd26--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/job/log/LogListener.java@@ -0,0 +1,7 @@+package io.onedev.server.job.log;++public interface LogListener {++ void logged(Long buildId);++}diff --git a/server-core/src/main/java/io/onedev/server/job/log/LogManager.java b/server-core/src/main/java/io/onedev/server/job/log/LogManager.javaindex f96ccc5703..fcbcbda0a0 100644--- a/server-core/src/main/java/io/onedev/server/job/log/LogManager.java+++ b/server-core/src/main/java/io/onedev/server/job/log/LogManager.java@@ -1,60 +1,64 @@-package io.onedev.server.job.log;--import java.io.InputStream;-import java.util.Collection;-import java.util.List;-import java.util.regex.Pattern;--import javax.annotation.Nullable;--import io.onedev.commons.utils.TaskLogger;-import io.onedev.server.buildspec.job.log.JobLogEntryEx;-import io.onedev.server.model.Build;--public interface LogManager {-- TaskLogger newLogger(Build build, Collection<String> jobSecrets);-- TaskLogger newLogger(Build build);-- /**- * Read specified number of log entries from specified build, starting from specified index- *- * @param build- * build to read log entries from- * @param offset- * index of the log entry to start read- * @param count- * number of log entries to read. Specifically use <tt>0</tt> to read all entries- * since offset- * @return- * log entries. Number of entries may be less than required count if there is no- * enough log entries- */- List<JobLogEntryEx> readLogEntries(Build build, int offset, int count);-- boolean matches(Build build, Pattern pattern);-- /**- * Read specified number of log entries starting from end of the log- *- * @param build- * build to read log entries from- * @param count- * number of log entries to read- * @return- * log entries with normal order. Number of entries may be less than required count- * if there is no enough log entries- */- LogSnippet readLogSnippetReversely(Build build, int count);-- InputStream openLogStream(Long projectId, Long buildNumber);-- @Nullable- TaskLogger getJobLogger(String jobToken);-- void addJobLogger(String jobToken, TaskLogger logger);-- void removeJobLogger(String jobToken);--}+package io.onedev.server.job.log;++import java.io.InputStream;+import java.util.Collection;+import java.util.List;+import java.util.regex.Pattern;++import javax.annotation.Nullable;++import io.onedev.commons.utils.TaskLogger;+import io.onedev.server.buildspec.job.log.JobLogEntryEx;+import io.onedev.server.model.Build;++public interface LogManager {++ TaskLogger newLogger(Build build, Collection<String> jobSecrets);++ TaskLogger newLogger(Build build);++ /**+ * Read specified number of log entries from specified build, starting from specified index+ *+ * @param build+ * build to read log entries from+ * @param offset+ * index of the log entry to start read+ * @param count+ * number of log entries to read. Specifically use <tt>0</tt> to read all entries+ * since offset+ * @return+ * log entries. Number of entries may be less than required count if there is no+ * enough log entries+ */+ List<JobLogEntryEx> readLogEntries(Long projectId, Long buildNumber, int offset, int count);++ void registerListener(LogListener listener);++ void deregisterListener(LogListener logListener);++ boolean matches(Build build, Pattern pattern);++ /**+ * Read specified number of log entries starting from end of the log+ *+ * @param build+ * build to read log entries from+ * @param count+ * number of log entries to read+ * @return+ * log entries with normal order. Number of entries may be less than required count+ * if there is no enough log entries+ */+ LogSnippet readLogSnippetReversely(Long projectId, Long buildNumber, int count);++ InputStream openLogStream(Long projectId, Long buildNumber);++ @Nullable+ TaskLogger getJobLogger(String jobToken);++ void addJobLogger(String jobToken, TaskLogger logger);++ void removeJobLogger(String jobToken);++}diff --git a/server-core/src/main/java/io/onedev/server/model/Build.java b/server-core/src/main/java/io/onedev/server/model/Build.javaindex 69e67616d0..64cfe09b48 100644--- a/server-core/src/main/java/io/onedev/server/model/Build.java+++ b/server-core/src/main/java/io/onedev/server/model/Build.java@@ -208,13 +208,61 @@ public class Build extends ProjectBelongingpublic enum Status {// Most significant status comes first, refer to getOverallStatus- WAITING, PENDING, RUNNING, FAILED, CANCELLED, TIMED_OUT, SUCCESSFUL;+ WAITING {+ @Override+ public boolean isFinished() {+ return false;+ }+ },+ PENDING {+ @Override+ public boolean isFinished() {+ return false;+ }++ },+ RUNNING {+ @Override+ public boolean isFinished() {+ return false;+ }++ },+ FAILED {+ @Override+ public boolean isFinished() {+ return true;+ }++ },+ CANCELLED {+ @Override+ public boolean isFinished() {+ return true;+ }++ },+ TIMED_OUT {+ @Override+ public boolean isFinished() {+ return true;+ }+ },+ SUCCESSFUL {+ @Override+ public boolean isFinished() {+ return true;+ }++ };@Overridepublic String toString() {return WordUtils.toWords(name());}+ public abstract boolean isFinished();+@Nullablepublic static Status of(String statusName) {for (Status status: Status.values()) {@@ -521,10 +569,7 @@ public class Build extends ProjectBelonging}public boolean isFinished() {- return status == Status.FAILED- || status == Status.CANCELLED- || status == Status.SUCCESSFUL- || status == Status.TIMED_OUT;+ return status.isFinished();}public boolean isSuccessful() {diff --git a/server-core/src/main/java/io/onedev/server/rest/resource/ArtifactResource.java b/server-core/src/main/java/io/onedev/server/rest/resource/ArtifactResource.javaindex a70f647d17..7c273f645b 100644--- a/server-core/src/main/java/io/onedev/server/rest/resource/ArtifactResource.java+++ b/server-core/src/main/java/io/onedev/server/rest/resource/ArtifactResource.java@@ -25,9 +25,9 @@ import javax.ws.rs.core.Response;import javax.ws.rs.core.StreamingOutput;import java.io.InputStream;-import static io.onedev.server.util.IOUtils.BUFFER_SIZE;import static io.onedev.k8shelper.KubernetesHelper.BEARER;import static io.onedev.k8shelper.KubernetesHelper.checkStatus;+import static io.onedev.server.util.IOUtils.BUFFER_SIZE;import static javax.ws.rs.client.Entity.entity;import static javax.ws.rs.core.HttpHeaders.AUTHORIZATION;import static javax.ws.rs.core.MediaType.APPLICATION_OCTET_STREAM;@@ -91,15 +91,16 @@ public class ArtifactResource {if (!SecurityUtils.canAccessBuild(build))throw new UnauthorizedException();- return output -> {- String activeServer = projectManager.getActiveServer(- build.getProject().getId(), true);+ var projectId = build.getProject().getId();+ var buildNumber = build.getNumber();+ return os -> {+ String activeServer = projectManager.getActiveServer(projectId, true);String serverUrl = clusterManager.getServerUrl(activeServer);Client client = ClientBuilder.newClient();try {WebTarget target = client.target(serverUrl).path("~api/cluster/artifact")- .queryParam("projectId", build.getProject().getId())- .queryParam("buildNumber", build.getNumber())+ .queryParam("projectId", projectId)+ .queryParam("buildNumber", buildNumber).queryParam("artifactPath", normalizeArtifactPath(artifactPath));Invocation.Builder builder = target.request();builder.header(AUTHORIZATION, BEARER + " "@@ -108,9 +109,9 @@ public class ArtifactResource {try (Response response = builder.get()) {checkStatus(response);try (InputStream is = response.readEntity(InputStream.class)) {- copy(is, output, BUFFER_SIZE);+ copy(is, os, BUFFER_SIZE);} finally {- output.close();+ os.close();}}} finally {@@ -131,8 +132,9 @@ public class ArtifactResource {if (!SecurityUtils.canManageBuild(build))throw new UnauthorizedException();- String activeServer = projectManager.getActiveServer(- build.getProject().getId(), true);+ var projectId = build.getProject().getId();+ var buildNumber = build.getNumber();+ String activeServer = projectManager.getActiveServer(projectId, true);String serverUrl = clusterManager.getServerUrl(activeServer);Client client = ClientBuilder.newClient();@@ -140,8 +142,8 @@ public class ArtifactResource {try {WebTarget target = client.target(serverUrl).path("~api/cluster/artifact")- .queryParam("projectId", build.getProject().getId())- .queryParam("buildNumber", build.getNumber())+ .queryParam("projectId", projectId)+ .queryParam("buildNumber", buildNumber).queryParam("artifactPath", normalizeArtifactPath(artifactPath));Invocation.Builder builder = target.request();builder.header(AUTHORIZATION, BEARER + " " + clusterManager.getCredential());diff --git a/server-core/src/main/java/io/onedev/server/rest/resource/BuildLogResource.java b/server-core/src/main/java/io/onedev/server/rest/resource/BuildLogResource.javanew file mode 100644index 0000000000..31da1804e7--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/rest/resource/BuildLogResource.java@@ -0,0 +1,153 @@+package io.onedev.server.rest.resource;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.onedev.server.buildspec.job.log.JobLogEntryEx;+import io.onedev.server.entitymanager.BuildManager;+import io.onedev.server.job.log.LogListener;+import io.onedev.server.job.log.LogManager;+import io.onedev.server.job.log.LogSnippet;+import io.onedev.server.model.Build;+import io.onedev.server.model.Build.Status;+import io.onedev.server.persistence.SessionManager;+import io.onedev.server.rest.annotation.Api;+import io.onedev.server.security.SecurityUtils;+import org.apache.shiro.authz.UnauthorizedException;++import javax.inject.Inject;+import javax.inject.Singleton;+import javax.ws.rs.*;+import javax.ws.rs.core.MediaType;+import javax.ws.rs.core.Response;+import javax.ws.rs.core.StreamingOutput;+import java.io.IOException;+import java.io.OutputStream;+import java.nio.ByteBuffer;++import static java.nio.charset.StandardCharsets.UTF_8;+import static javax.ws.rs.core.MediaType.APPLICATION_OCTET_STREAM;++@Api(order=4200, description="In most cases, build log resource is operated with build id, which is different from build number. "+ + "To get build id of a particular build number, use the <a href='/~help/api/io.onedev.server.rest.BuildResource/queryBasicInfo'>Query Basic Info</a> operation with query for "+ + "instance <code>"Number" is "projectName#100"</code>")+@Path("/build-logs")+@Consumes(MediaType.APPLICATION_JSON)+@Produces(MediaType.APPLICATION_JSON)+@Singleton+public class BuildLogResource {++ private static final int MAX_LOG_ENTRIES = 1000;++ private final BuildManager buildManager;++ private final LogManager logManager;++ private final ObjectMapper objectMapper;++ private final SessionManager sessionManager;++ @Inject+ public BuildLogResource(BuildManager buildManager, LogManager logManager,+ ObjectMapper objectMapper, SessionManager sessionManager) {+ this.buildManager = buildManager;+ this.logManager = logManager;+ this.objectMapper = objectMapper;+ this.sessionManager = sessionManager;+ }++ @Api(order=200, description = "Streaming build log of specified build")+ @Path("/{buildId}/streaming")+ @GET+ @Produces(APPLICATION_OCTET_STREAM)+ public StreamingOutput downloadLog(@PathParam("buildId") Long buildId) {+ Build build = buildManager.load(buildId);+ if (!SecurityUtils.canAccessLog(build))+ throw new UnauthorizedException();++ var projectId = build.getProject().getId();+ var buildNumber = build.getNumber();+ var buildStatus = build.getStatus();++ return os -> {+ writeStatus(os, buildStatus);+ var logListener = new LogListener() {++ @Override+ public void logged(Long loggedBuildId) {+ if (loggedBuildId.equals(buildId)) {+ synchronized (os) {+ os.notify();+ }+ }+ }++ };+ logManager.registerListener(logListener);+ sessionManager.closeSession();+ try {+ var nextOffset = 0;+ LogSnippet snippet = logManager.readLogSnippetReversely(projectId, buildNumber, MAX_LOG_ENTRIES + 1);+ nextOffset = snippet.offset + snippet.entries.size();+ for (var entry : snippet.entries)+ writeEntry(os, entry);++ while (true) {+ synchronized (os) {+ try {+ os.wait(5000);+ } catch (InterruptedException e) {+ throw new RuntimeException(e);+ }+ var entries = logManager.readLogEntries(projectId, buildNumber, nextOffset, 0);+ if (!entries.isEmpty()) {+ nextOffset += entries.size();+ for (var entry : entries)+ writeEntry(os, entry);+ } else {+ var innerBuildStatus = sessionManager.call(() -> buildManager.load(buildId).getStatus());+ if (innerBuildStatus.isFinished()) {+ writeStatus(os, innerBuildStatus);+ break;+ } else {+ writeInt(os, 0);+ os.flush();+ }+ }+ }+ }+ } finally {+ sessionManager.openSession();+ logManager.deregisterListener(logListener);+ }+ };+ }++ private void writeInt(OutputStream os, int value) {+ try {+ os.write(ByteBuffer.allocate(Integer.BYTES).putInt(value).array());+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }++ private void writeStatus(OutputStream os, Status status) {+ try {+ writeInt(os, status.name().length() * -1);+ os.write(status.name().getBytes(UTF_8));+ os.flush();+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }++ private void writeEntry(OutputStream os, JobLogEntryEx entry) {+ try {+ var bytes = objectMapper.writeValueAsBytes(entry.transformEmojis());+ writeInt(os, bytes.length);+ os.write(bytes);+ os.flush();+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }++}diff --git a/server-core/src/main/java/io/onedev/server/rest/resource/SettingResource.java b/server-core/src/main/java/io/onedev/server/rest/resource/SettingResource.javaindex 7c83fafe9c..d7f1c28c88 100644--- a/server-core/src/main/java/io/onedev/server/rest/resource/SettingResource.java+++ b/server-core/src/main/java/io/onedev/server/rest/resource/SettingResource.java@@ -23,7 +23,7 @@ import javax.ws.rs.core.Response;import java.util.List;import java.util.Map;-@Api(order=100000)+@Api(order=10000)@Path("/settings")@Consumes(MediaType.APPLICATION_JSON)@Produces(MediaType.APPLICATION_JSON)diff --git a/server-core/src/main/java/io/onedev/server/rest/resource/VersionResource.java b/server-core/src/main/java/io/onedev/server/rest/resource/VersionResource.javanew file mode 100644index 0000000000..141db18541--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/rest/resource/VersionResource.java@@ -0,0 +1,56 @@+package io.onedev.server.rest.resource;++import io.onedev.server.OneDev;+import io.onedev.server.rest.annotation.Api;++import javax.inject.Singleton;+import javax.ws.rs.*;+import javax.ws.rs.core.MediaType;+import java.io.Serializable;++@Api(order=100000, description = "Version info for server and various tools")+@Path("/version")+@Produces(MediaType.APPLICATION_JSON)+@Singleton+public class VersionResource {++ @Api(order=100, description = "Get server version")+ @Path("/server")+ @GET+ public String getServerVersion() {+ return OneDev.getInstance().getVersion();+ }++ @Api(description = "Get tod version range compatible with this server")+ @Path("/compatible-tod-versions")+ @GET+ public VersionRange getCompatibleTodVersions() {+ return new VersionRange();+ }++ private static class VersionRange implements Serializable {++ @Api(example = "1.0.0", description = "Minimum version. Null means no lower bound")+ String minVersion;++ @Api(example = "2.0.0", description = "Maximum version. Null means no upper bound")+ String maxVersion;++ public String getMinVersion() {+ return minVersion;+ }++ public void setMinVersion(String minVersion) {+ this.minVersion = minVersion;+ }++ public String getMaxVersion() {+ return maxVersion;+ }++ public void setMaxVersion(String maxVersion) {+ this.maxVersion = maxVersion;+ }+ }++}diff --git a/server-core/src/main/java/io/onedev/server/web/component/build/log/BuildLogPanel.java b/server-core/src/main/java/io/onedev/server/web/component/build/log/BuildLogPanel.javaindex 667af75661..e7937d3810 100644--- a/server-core/src/main/java/io/onedev/server/web/component/build/log/BuildLogPanel.java+++ b/server-core/src/main/java/io/onedev/server/web/component/build/log/BuildLogPanel.java@@ -1,36 +1,32 @@package io.onedev.server.web.component.build.log;-import java.util.ArrayList;-import java.util.Collection;-import java.util.List;--import io.onedev.server.web.page.base.BasePage;-import org.apache.wicket.ajax.AjaxRequestTarget;-import org.apache.wicket.core.request.handler.IPartialPageRequestHandler;-import org.apache.wicket.markup.head.IHeaderResponse;-import org.apache.wicket.markup.head.JavaScriptHeaderItem;-import org.apache.wicket.markup.head.OnDomReadyHeaderItem;-import org.apache.wicket.markup.html.panel.GenericPanel;-import org.apache.wicket.model.IModel;-import com.fasterxml.jackson.core.JsonProcessingException;import com.fasterxml.jackson.databind.ObjectMapper;import com.google.common.collect.Sets;-import io.onedev.server.OneDev;import io.onedev.server.buildspec.job.log.JobLogEntryEx;-import io.onedev.server.buildspec.job.log.Message;import io.onedev.server.job.JobManager;import io.onedev.server.job.log.LogManager;import io.onedev.server.job.log.LogSnippet;import io.onedev.server.model.Build;import io.onedev.server.model.Build.Status;import io.onedev.server.security.SecurityUtils;-import io.onedev.server.web.asset.emoji.Emojis;import io.onedev.server.web.behavior.AbstractPostAjaxBehavior;import io.onedev.server.web.behavior.ChangeObserver;+import io.onedev.server.web.page.base.BasePage;+import org.apache.wicket.ajax.AjaxRequestTarget;+import org.apache.wicket.core.request.handler.IPartialPageRequestHandler;+import org.apache.wicket.markup.head.IHeaderResponse;+import org.apache.wicket.markup.head.JavaScriptHeaderItem;+import org.apache.wicket.markup.head.OnDomReadyHeaderItem;+import org.apache.wicket.markup.html.panel.GenericPanel;+import org.apache.wicket.model.IModel;++import java.util.Collection;+import java.util.List;import static io.onedev.server.model.Build.getDetailChangeObservable;+import static java.util.stream.Collectors.toList;@SuppressWarnings("serial")public class BuildLogPanel extends GenericPanel<Build> {@@ -52,7 +48,8 @@ public class BuildLogPanel extends GenericPanel<Build> {add(new ChangeObserver() {private void appendRecentLogEntries(IPartialPageRequestHandler handler) {- List<JobLogEntryEx> logEntries = getLogManager().readLogEntries(getBuild(), nextOffset, 0);+ List<JobLogEntryEx> logEntries = getLogManager().readLogEntries(+ getBuild().getProject().getId(), getBuild().getNumber(), nextOffset, 0);if (!logEntries.isEmpty()) {nextOffset += logEntries.size();@@ -113,16 +110,9 @@ public class BuildLogPanel extends GenericPanel<Build> {}private String asJSON(List<JobLogEntryEx> entries) {- List<JobLogEntryEx> transformed = new ArrayList<>();- Emojis emojis = Emojis.getInstance();- for (JobLogEntryEx entry: entries) {- List<Message> messages = new ArrayList<>();- for (Message message: entry.getMessages())- messages.add(new Message(message.getStyle(), emojis.apply(message.getText())));- transformed.add(new JobLogEntryEx(entry.getDate(), messages));- }+ entries = entries.stream().map(JobLogEntryEx::transformEmojis).collect(toList());try {- return OneDev.getInstance(ObjectMapper.class).writeValueAsString(transformed);+ return OneDev.getInstance(ObjectMapper.class).writeValueAsString(entries);} catch (JsonProcessingException e) {throw new RuntimeException(e);}@@ -134,7 +124,8 @@ public class BuildLogPanel extends GenericPanel<Build> {response.render(JavaScriptHeaderItem.forReference(new BuildLogResourceReference()));- LogSnippet snippet = getLogManager().readLogSnippetReversely(getBuild(), MAX_LOG_ENTRIES+1);+ LogSnippet snippet = getLogManager().readLogSnippetReversely(+ getBuild().getProject().getId(), getBuild().getNumber(), MAX_LOG_ENTRIES+1);nextOffset = snippet.offset + snippet.entries.size();diff --git a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.javaindex aa19d9c3cc..5b2ea791e9 100644--- a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java+++ b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java@@ -83,94 +83,86 @@ public class KubernetesResource {@Path("/run-server-step")@Consumes(MediaType.APPLICATION_OCTET_STREAM)@POST- public Response runServerStep(@QueryParam("jobToken") String jobToken, InputStream is) {+ public StreamingOutput runServerStep(@QueryParam("jobToken") String jobToken, InputStream is) {JobContext jobContext = jobManager.getJobContext(jobToken, true);- // Make sure we are not occupying a database connection here as we will occupy- // database connection when running step at project server side- sessionManager.closeSession();- try {- StreamingOutput os = output -> {- File filesDir = FileUtils.createTempDir();- try {- int length = readInt(is);- List<Integer> stepPosition = new ArrayList<>();- for (int i=0; i<length; i++)- stepPosition.add(readInt(is));-- Map<String, String> placeholderValues = new HashMap<>();- length = readInt(is);- for (int i=0; i<length; i++)- placeholderValues.put(readString(is), readString(is));-- TarUtils.untar(is, filesDir, false);-- Map<String, byte[]> outputFiles = jobManager.runServerStep(jobContext,- stepPosition, filesDir, placeholderValues, true, new TaskLogger() {+ return os -> {+ File filesDir = FileUtils.createTempDir();+ // Make sure we are not occupying a database connection here as we will occupy+ // database connection when running step at project server side+ sessionManager.closeSession();+ try {+ int length = readInt(is);+ List<Integer> stepPosition = new ArrayList<>();+ for (int i=0; i<length; i++)+ stepPosition.add(readInt(is));++ Map<String, String> placeholderValues = new HashMap<>();+ length = readInt(is);+ for (int i=0; i<length; i++)+ placeholderValues.put(readString(is), readString(is));++ TarUtils.untar(is, filesDir, false);++ Map<String, byte[]> outputFiles = jobManager.runServerStep(jobContext,+ stepPosition, filesDir, placeholderValues, true, new TaskLogger() {- @Override- public void log(String message, String sessionId) {- // While testing, ngrok.io buffers response and build can not get log entries- // timely. This won't happen on pagekite however- KubernetesHelper.writeInt(output, 1);- KubernetesHelper.writeString(output, message);- try {- output.flush();- } catch (IOException e) {- throw new RuntimeException(e);- }+ @Override+ public void log(String message, String sessionId) {+ // While testing, ngrok.io buffers response and build can not get log entries+ // timely. This won't happen on pagekite however+ KubernetesHelper.writeInt(os, 1);+ KubernetesHelper.writeString(os, message);+ try {+ os.flush();+ } catch (IOException e) {+ throw new RuntimeException(e);}-- });- if (outputFiles == null)- outputFiles = new HashMap<>();- byte[] bytes = SerializationUtils.serialize((Serializable) outputFiles);- KubernetesHelper.writeInt(output, 2);- KubernetesHelper.writeInt(output, bytes.length);- output.write(bytes);- } finally {- FileUtils.deleteDir(filesDir);- }- };- return Response.ok(os).build();- } finally {- sessionManager.openSession();- }+ }++ });+ if (outputFiles == null)+ outputFiles = new HashMap<>();+ byte[] bytes = SerializationUtils.serialize((Serializable) outputFiles);+ KubernetesHelper.writeInt(os, 2);+ KubernetesHelper.writeInt(os, bytes.length);+ os.write(bytes);+ } finally {+ sessionManager.openSession();+ FileUtils.deleteDir(filesDir);+ }+ };}@Path("/download-dependencies")@Produces(MediaType.APPLICATION_OCTET_STREAM)@GET- public Response downloadDependencies(@QueryParam("jobToken") String jobToken) {- sessionManager.closeSession();- try {- StreamingOutput output = os -> {- JobContext jobContext = jobManager.getJobContext(jobToken, true);- File tempDir = FileUtils.createTempDir();- try {- jobManager.copyDependencies(jobContext, tempDir);- TarUtils.tar(tempDir, os, false);- os.flush();- } finally {- FileUtils.deleteDir(tempDir);- }- };- return Response.ok(output).build();- } finally {- sessionManager.openSession();- }+ public StreamingOutput downloadDependencies(@QueryParam("jobToken") String jobToken) {+ return os -> {+ JobContext jobContext = jobManager.getJobContext(jobToken, true);+ File tempDir = FileUtils.createTempDir();+ sessionManager.closeSession();+ try {+ jobManager.copyDependencies(jobContext, tempDir);+ TarUtils.tar(tempDir, os, false);+ os.flush();+ } finally {+ sessionManager.openSession();+ FileUtils.deleteDir(tempDir);+ }+ };}@Path("/download-cache")@Produces(MediaType.APPLICATION_OCTET_STREAM)@GET- public Response downloadCache(+ public StreamingOutput downloadCache(@QueryParam("jobToken") String jobToken,@QueryParam("cacheKey") @Nullable String cacheKey,@QueryParam("cacheLoadKeys") @Nullable String joinedCacheLoadKeys,@QueryParam("cachePath") String cachePath) {- sessionManager.closeSession();- try {- StreamingOutput output = os -> {+ return os -> {+ sessionManager.closeSession();+ try {var jobContext = jobManager.getJobContext(jobToken, true);if (cacheKey != null) {jobCacheManager.downloadCache(jobContext.getProjectId(), cacheKey, cachePath, os);@@ -178,11 +170,10 @@ public class KubernetesResource {var cacheLoadKeys = Splitter.on('\n').splitToList(joinedCacheLoadKeys);jobCacheManager.downloadCache(jobContext.getProjectId(), cacheLoadKeys, cachePath, os);}- };- return Response.ok(output).build();- } finally {- sessionManager.openSession();- }+ } finally {+ sessionManager.openSession();+ }+ };}@Path("/upload-cache")