Skip to content

The Unified and Autonomous Development Platform

chore: Build log streaming restful api

Robin Shenec0db843eafb
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/job/log/JobLogEntryEx.java b/server-core/src/main/java/io/onedev/server/buildspec/job/log/JobLogEntryEx.java
index 7520366bea..aff9b8c37e 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/job/log/JobLogEntryEx.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/job/log/JobLogEntryEx.java
@@ -14,6 +14,7 @@ import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
import io.onedev.server.job.log.StyleBuilder;
+import io.onedev.server.web.asset.emoji.Emojis;
public class JobLogEntryEx implements Serializable {
@@ -32,6 +33,14 @@ public class JobLogEntryEx implements Serializable {
this(entry.getDate(), Lists.newArrayList(new Message(new StyleBuilder().build(), entry.getMessage())));
}
+ public JobLogEntryEx transformEmojis() {
+ Emojis emojis = Emojis.getInstance();
+ List<Message> messages = new ArrayList<>();
+ for (Message message: getMessages())
+ messages.add(new Message(message.getStyle(), emojis.apply(message.getText())));
+ return new JobLogEntryEx(getDate(), messages);
+ }
+
// Handle ANSI escape codes according to https://en.wikipedia.org/wiki/ANSI_escape_code
public static JobLogEntryEx parse(String text, StyleBuilder styleBuilder) {
text = text.replace("\r\n", "\n");
diff --git a/server-core/src/main/java/io/onedev/server/job/log/DefaultLogManager.java b/server-core/src/main/java/io/onedev/server/job/log/DefaultLogManager.java
index 4a2b5105b6..6f5aae5e31 100644
--- a/server-core/src/main/java/io/onedev/server/job/log/DefaultLogManager.java
+++ b/server-core/src/main/java/io/onedev/server/job/log/DefaultLogManager.java
@@ -11,12 +11,14 @@ import io.onedev.server.buildspec.job.log.instruction.LogInstruction;
import io.onedev.server.buildspec.job.log.instruction.LogInstructionParser.InstructionContext;
import io.onedev.server.buildspec.job.log.instruction.LogInstructionParser.ParamContext;
import io.onedev.server.buildspecmodel.inputspec.SecretInput;
+import io.onedev.server.cluster.ClusterManager;
import io.onedev.server.cluster.ClusterTask;
import io.onedev.server.entitymanager.BuildManager;
import io.onedev.server.entitymanager.ProjectManager;
import io.onedev.server.event.Listen;
import io.onedev.server.event.project.build.BuildFinished;
import io.onedev.server.model.Build;
+import io.onedev.server.persistence.TransactionManager;
import io.onedev.server.persistence.annotation.Sessional;
import io.onedev.server.web.websocket.WebSocketManager;
import org.antlr.v4.runtime.tree.TerminalNode;
@@ -34,6 +36,8 @@ import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Pattern;
import static io.onedev.commons.utils.LockUtils.*;
@@ -58,15 +62,27 @@ public class DefaultLogManager implements LogManager, Serializable {
private final BuildManager buildManager;
+ private final ClusterManager clusterManager;
+
+ private final TransactionManager transactionManager;
+
private final Map<String, LogSnippet> recentSnippets = new ConcurrentHashMap<>();
private final Map<String, TaskLogger> jobLoggers = new ConcurrentHashMap<>();
+ private final ReadWriteLock logListenersLock = new ReentrantReadWriteLock();
+
+ private final List<LogListener> logListeners = new ArrayList<>();
+
@Inject
- public DefaultLogManager(WebSocketManager webSocketManager, ProjectManager projectManager, BuildManager buildManager) {
+ public DefaultLogManager(WebSocketManager webSocketManager, ProjectManager projectManager,
+ BuildManager buildManager, ClusterManager clusterManager,
+ TransactionManager transactionManager) {
this.projectManager = projectManager;
this.webSocketManager = webSocketManager;
this.buildManager = buildManager;
+ this.clusterManager = clusterManager;
+ this.transactionManager = transactionManager;
}
public Object writeReplace() throws ObjectStreamException {
@@ -78,6 +94,20 @@ public class DefaultLogManager implements LogManager, Serializable {
return newLogger(build, new ArrayList<>());
}
+ private void notifyListeners(Long buildId) {
+ clusterManager.submitToAllServers(() -> {
+ var lock = logListenersLock.readLock();
+ lock.lock();
+ try {
+ for (var logListener: logListeners)
+ logListener.logged(buildId);
+ } finally {
+ lock.unlock();
+ }
+ return null;
+ });
+ }
+
@Override
public TaskLogger newLogger(Build build, Collection<String> jobSecretsToMask) {
Long projectId = build.getProject().getId();
@@ -125,6 +155,7 @@ public class DefaultLogManager implements LogManager, Serializable {
}
}
webSocketManager.notifyObservableChange(Build.getLogChangeObservable(buildId), null);
+ notifyListeners(buildId);
}
}
return null;
@@ -291,11 +322,8 @@ public class DefaultLogManager implements LogManager, Serializable {
}
}
- @Sessional
@Override
- public List<JobLogEntryEx> readLogEntries(Build build, int from, int count) {
- Long projectId = build.getProject().getId();
- Long buildNumber = build.getNumber();
+ public List<JobLogEntryEx> readLogEntries(Long projectId, Long buildNumber, int from, int count) {
return projectManager.runOnActiveServer(projectId, new ClusterTask<>() {
private static final long serialVersionUID = 1L;
@@ -325,11 +353,8 @@ public class DefaultLogManager implements LogManager, Serializable {
});
}
- @Sessional
@Override
- public LogSnippet readLogSnippetReversely(Build build, int count) {
- Long projectId = build.getProject().getId();
- Long buildNumber = build.getNumber();
+ public LogSnippet readLogSnippetReversely(Long projectId, Long buildNumber, int count) {
return projectManager.runOnActiveServer(projectId, new ClusterTask<>() {
private static final long serialVersionUID = 1L;
@@ -397,6 +422,7 @@ public class DefaultLogManager implements LogManager, Serializable {
}
return null;
});
+ transactionManager.runAfterCommit(() -> notifyListeners(build.getId()));
}
private String getLogKey(Long projectId, Long buildNumber) {
@@ -501,6 +527,28 @@ public class DefaultLogManager implements LogManager, Serializable {
}
+ @Override
+ public void registerListener(LogListener listener) {
+ var lock = logListenersLock.writeLock();
+ lock.lock();
+ try {
+ logListeners.add(listener);
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public void deregisterListener(LogListener listener) {
+ var lock = logListenersLock.writeLock();
+ lock.lock();
+ try {
+ logListeners.remove(listener);
+ } finally {
+ lock.unlock();
+ }
+ }
+
@Override
public TaskLogger getJobLogger(String jobToken) {
return jobLoggers.get(jobToken);
diff --git a/server-core/src/main/java/io/onedev/server/job/log/LogListener.java b/server-core/src/main/java/io/onedev/server/job/log/LogListener.java
new file mode 100644
index 0000000000..5f63dbbd26
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/job/log/LogListener.java
@@ -0,0 +1,7 @@
+package io.onedev.server.job.log;
+
+public interface LogListener {
+
+ void logged(Long buildId);
+
+}
diff --git a/server-core/src/main/java/io/onedev/server/job/log/LogManager.java b/server-core/src/main/java/io/onedev/server/job/log/LogManager.java
index f96ccc5703..fcbcbda0a0 100644
--- a/server-core/src/main/java/io/onedev/server/job/log/LogManager.java
+++ b/server-core/src/main/java/io/onedev/server/job/log/LogManager.java
@@ -1,60 +1,64 @@
-package io.onedev.server.job.log;
-
-import java.io.InputStream;
-import java.util.Collection;
-import java.util.List;
-import java.util.regex.Pattern;
-
-import javax.annotation.Nullable;
-
-import io.onedev.commons.utils.TaskLogger;
-import io.onedev.server.buildspec.job.log.JobLogEntryEx;
-import io.onedev.server.model.Build;
-
-public interface LogManager {
-
- TaskLogger newLogger(Build build, Collection<String> jobSecrets);
-
- TaskLogger newLogger(Build build);
-
- /**
- * Read specified number of log entries from specified build, starting from specified index
- *
- * @param build
- * build to read log entries from
- * @param offset
- * index of the log entry to start read
- * @param count
- * number of log entries to read. Specifically use <tt>0</tt> to read all entries
- * since offset
- * @return
- * log entries. Number of entries may be less than required count if there is no
- * enough log entries
- */
- List<JobLogEntryEx> readLogEntries(Build build, int offset, int count);
-
- boolean matches(Build build, Pattern pattern);
-
- /**
- * Read specified number of log entries starting from end of the log
- *
- * @param build
- * build to read log entries from
- * @param count
- * number of log entries to read
- * @return
- * log entries with normal order. Number of entries may be less than required count
- * if there is no enough log entries
- */
- LogSnippet readLogSnippetReversely(Build build, int count);
-
- InputStream openLogStream(Long projectId, Long buildNumber);
-
- @Nullable
- TaskLogger getJobLogger(String jobToken);
-
- void addJobLogger(String jobToken, TaskLogger logger);
-
- void removeJobLogger(String jobToken);
-
-}
+package io.onedev.server.job.log;
+
+import java.io.InputStream;
+import java.util.Collection;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import javax.annotation.Nullable;
+
+import io.onedev.commons.utils.TaskLogger;
+import io.onedev.server.buildspec.job.log.JobLogEntryEx;
+import io.onedev.server.model.Build;
+
+public interface LogManager {
+
+ TaskLogger newLogger(Build build, Collection<String> jobSecrets);
+
+ TaskLogger newLogger(Build build);
+
+ /**
+ * Read specified number of log entries from specified build, starting from specified index
+ *
+ * @param build
+ * build to read log entries from
+ * @param offset
+ * index of the log entry to start read
+ * @param count
+ * number of log entries to read. Specifically use <tt>0</tt> to read all entries
+ * since offset
+ * @return
+ * log entries. Number of entries may be less than required count if there is no
+ * enough log entries
+ */
+ List<JobLogEntryEx> readLogEntries(Long projectId, Long buildNumber, int offset, int count);
+
+ void registerListener(LogListener listener);
+
+ void deregisterListener(LogListener logListener);
+
+ boolean matches(Build build, Pattern pattern);
+
+ /**
+ * Read specified number of log entries starting from end of the log
+ *
+ * @param build
+ * build to read log entries from
+ * @param count
+ * number of log entries to read
+ * @return
+ * log entries with normal order. Number of entries may be less than required count
+ * if there is no enough log entries
+ */
+ LogSnippet readLogSnippetReversely(Long projectId, Long buildNumber, int count);
+
+ InputStream openLogStream(Long projectId, Long buildNumber);
+
+ @Nullable
+ TaskLogger getJobLogger(String jobToken);
+
+ void addJobLogger(String jobToken, TaskLogger logger);
+
+ void removeJobLogger(String jobToken);
+
+}
diff --git a/server-core/src/main/java/io/onedev/server/model/Build.java b/server-core/src/main/java/io/onedev/server/model/Build.java
index 69e67616d0..64cfe09b48 100644
--- a/server-core/src/main/java/io/onedev/server/model/Build.java
+++ b/server-core/src/main/java/io/onedev/server/model/Build.java
@@ -208,13 +208,61 @@ public class Build extends ProjectBelonging
public enum Status {
// Most significant status comes first, refer to getOverallStatus
- WAITING, PENDING, RUNNING, FAILED, CANCELLED, TIMED_OUT, SUCCESSFUL;
+ WAITING {
+ @Override
+ public boolean isFinished() {
+ return false;
+ }
+ },
+ PENDING {
+ @Override
+ public boolean isFinished() {
+ return false;
+ }
+
+ },
+ RUNNING {
+ @Override
+ public boolean isFinished() {
+ return false;
+ }
+
+ },
+ FAILED {
+ @Override
+ public boolean isFinished() {
+ return true;
+ }
+
+ },
+ CANCELLED {
+ @Override
+ public boolean isFinished() {
+ return true;
+ }
+
+ },
+ TIMED_OUT {
+ @Override
+ public boolean isFinished() {
+ return true;
+ }
+ },
+ SUCCESSFUL {
+ @Override
+ public boolean isFinished() {
+ return true;
+ }
+
+ };
@Override
public String toString() {
return WordUtils.toWords(name());
}
+ public abstract boolean isFinished();
+
@Nullable
public static Status of(String statusName) {
for (Status status: Status.values()) {
@@ -521,10 +569,7 @@ public class Build extends ProjectBelonging
}
public boolean isFinished() {
- return status == Status.FAILED
- || status == Status.CANCELLED
- || status == Status.SUCCESSFUL
- || status == Status.TIMED_OUT;
+ return status.isFinished();
}
public boolean isSuccessful() {
diff --git a/server-core/src/main/java/io/onedev/server/rest/resource/ArtifactResource.java b/server-core/src/main/java/io/onedev/server/rest/resource/ArtifactResource.java
index a70f647d17..7c273f645b 100644
--- a/server-core/src/main/java/io/onedev/server/rest/resource/ArtifactResource.java
+++ b/server-core/src/main/java/io/onedev/server/rest/resource/ArtifactResource.java
@@ -25,9 +25,9 @@ import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput;
import java.io.InputStream;
-import static io.onedev.server.util.IOUtils.BUFFER_SIZE;
import static io.onedev.k8shelper.KubernetesHelper.BEARER;
import static io.onedev.k8shelper.KubernetesHelper.checkStatus;
+import static io.onedev.server.util.IOUtils.BUFFER_SIZE;
import static javax.ws.rs.client.Entity.entity;
import static javax.ws.rs.core.HttpHeaders.AUTHORIZATION;
import static javax.ws.rs.core.MediaType.APPLICATION_OCTET_STREAM;
@@ -91,15 +91,16 @@ public class ArtifactResource {
if (!SecurityUtils.canAccessBuild(build))
throw new UnauthorizedException();
- return output -> {
- String activeServer = projectManager.getActiveServer(
- build.getProject().getId(), true);
+ var projectId = build.getProject().getId();
+ var buildNumber = build.getNumber();
+ return os -> {
+ String activeServer = projectManager.getActiveServer(projectId, true);
String serverUrl = clusterManager.getServerUrl(activeServer);
Client client = ClientBuilder.newClient();
try {
WebTarget target = client.target(serverUrl).path("~api/cluster/artifact")
- .queryParam("projectId", build.getProject().getId())
- .queryParam("buildNumber", build.getNumber())
+ .queryParam("projectId", projectId)
+ .queryParam("buildNumber", buildNumber)
.queryParam("artifactPath", normalizeArtifactPath(artifactPath));
Invocation.Builder builder = target.request();
builder.header(AUTHORIZATION, BEARER + " "
@@ -108,9 +109,9 @@ public class ArtifactResource {
try (Response response = builder.get()) {
checkStatus(response);
try (InputStream is = response.readEntity(InputStream.class)) {
- copy(is, output, BUFFER_SIZE);
+ copy(is, os, BUFFER_SIZE);
} finally {
- output.close();
+ os.close();
}
}
} finally {
@@ -131,8 +132,9 @@ public class ArtifactResource {
if (!SecurityUtils.canManageBuild(build))
throw new UnauthorizedException();
- String activeServer = projectManager.getActiveServer(
- build.getProject().getId(), true);
+ var projectId = build.getProject().getId();
+ var buildNumber = build.getNumber();
+ String activeServer = projectManager.getActiveServer(projectId, true);
String serverUrl = clusterManager.getServerUrl(activeServer);
Client client = ClientBuilder.newClient();
@@ -140,8 +142,8 @@ public class ArtifactResource {
try {
WebTarget target = client.target(serverUrl)
.path("~api/cluster/artifact")
- .queryParam("projectId", build.getProject().getId())
- .queryParam("buildNumber", build.getNumber())
+ .queryParam("projectId", projectId)
+ .queryParam("buildNumber", buildNumber)
.queryParam("artifactPath", normalizeArtifactPath(artifactPath));
Invocation.Builder builder = target.request();
builder.header(AUTHORIZATION, BEARER + " " + clusterManager.getCredential());
diff --git a/server-core/src/main/java/io/onedev/server/rest/resource/BuildLogResource.java b/server-core/src/main/java/io/onedev/server/rest/resource/BuildLogResource.java
new file mode 100644
index 0000000000..31da1804e7
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/rest/resource/BuildLogResource.java
@@ -0,0 +1,153 @@
+package io.onedev.server.rest.resource;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.onedev.server.buildspec.job.log.JobLogEntryEx;
+import io.onedev.server.entitymanager.BuildManager;
+import io.onedev.server.job.log.LogListener;
+import io.onedev.server.job.log.LogManager;
+import io.onedev.server.job.log.LogSnippet;
+import io.onedev.server.model.Build;
+import io.onedev.server.model.Build.Status;
+import io.onedev.server.persistence.SessionManager;
+import io.onedev.server.rest.annotation.Api;
+import io.onedev.server.security.SecurityUtils;
+import org.apache.shiro.authz.UnauthorizedException;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import javax.ws.rs.*;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.StreamingOutput;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static javax.ws.rs.core.MediaType.APPLICATION_OCTET_STREAM;
+
+@Api(order=4200, description="In most cases, build log resource is operated with build id, which is different from build number. "
+ + "To get build id of a particular build number, use the <a href='/~help/api/io.onedev.server.rest.BuildResource/queryBasicInfo'>Query Basic Info</a> operation with query for "
+ + "instance <code>&quot;Number&quot; is &quot;projectName#100&quot;</code>")
+@Path("/build-logs")
+@Consumes(MediaType.APPLICATION_JSON)
+@Produces(MediaType.APPLICATION_JSON)
+@Singleton
+public class BuildLogResource {
+
+ private static final int MAX_LOG_ENTRIES = 1000;
+
+ private final BuildManager buildManager;
+
+ private final LogManager logManager;
+
+ private final ObjectMapper objectMapper;
+
+ private final SessionManager sessionManager;
+
+ @Inject
+ public BuildLogResource(BuildManager buildManager, LogManager logManager,
+ ObjectMapper objectMapper, SessionManager sessionManager) {
+ this.buildManager = buildManager;
+ this.logManager = logManager;
+ this.objectMapper = objectMapper;
+ this.sessionManager = sessionManager;
+ }
+
+ @Api(order=200, description = "Streaming build log of specified build")
+ @Path("/{buildId}/streaming")
+ @GET
+ @Produces(APPLICATION_OCTET_STREAM)
+ public StreamingOutput downloadLog(@PathParam("buildId") Long buildId) {
+ Build build = buildManager.load(buildId);
+ if (!SecurityUtils.canAccessLog(build))
+ throw new UnauthorizedException();
+
+ var projectId = build.getProject().getId();
+ var buildNumber = build.getNumber();
+ var buildStatus = build.getStatus();
+
+ return os -> {
+ writeStatus(os, buildStatus);
+ var logListener = new LogListener() {
+
+ @Override
+ public void logged(Long loggedBuildId) {
+ if (loggedBuildId.equals(buildId)) {
+ synchronized (os) {
+ os.notify();
+ }
+ }
+ }
+
+ };
+ logManager.registerListener(logListener);
+ sessionManager.closeSession();
+ try {
+ var nextOffset = 0;
+ LogSnippet snippet = logManager.readLogSnippetReversely(projectId, buildNumber, MAX_LOG_ENTRIES + 1);
+ nextOffset = snippet.offset + snippet.entries.size();
+ for (var entry : snippet.entries)
+ writeEntry(os, entry);
+
+ while (true) {
+ synchronized (os) {
+ try {
+ os.wait(5000);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ var entries = logManager.readLogEntries(projectId, buildNumber, nextOffset, 0);
+ if (!entries.isEmpty()) {
+ nextOffset += entries.size();
+ for (var entry : entries)
+ writeEntry(os, entry);
+ } else {
+ var innerBuildStatus = sessionManager.call(() -> buildManager.load(buildId).getStatus());
+ if (innerBuildStatus.isFinished()) {
+ writeStatus(os, innerBuildStatus);
+ break;
+ } else {
+ writeInt(os, 0);
+ os.flush();
+ }
+ }
+ }
+ }
+ } finally {
+ sessionManager.openSession();
+ logManager.deregisterListener(logListener);
+ }
+ };
+ }
+
+ private void writeInt(OutputStream os, int value) {
+ try {
+ os.write(ByteBuffer.allocate(Integer.BYTES).putInt(value).array());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void writeStatus(OutputStream os, Status status) {
+ try {
+ writeInt(os, status.name().length() * -1);
+ os.write(status.name().getBytes(UTF_8));
+ os.flush();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void writeEntry(OutputStream os, JobLogEntryEx entry) {
+ try {
+ var bytes = objectMapper.writeValueAsBytes(entry.transformEmojis());
+ writeInt(os, bytes.length);
+ os.write(bytes);
+ os.flush();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+}
diff --git a/server-core/src/main/java/io/onedev/server/rest/resource/SettingResource.java b/server-core/src/main/java/io/onedev/server/rest/resource/SettingResource.java
index 7c83fafe9c..d7f1c28c88 100644
--- a/server-core/src/main/java/io/onedev/server/rest/resource/SettingResource.java
+++ b/server-core/src/main/java/io/onedev/server/rest/resource/SettingResource.java
@@ -23,7 +23,7 @@ import javax.ws.rs.core.Response;
import java.util.List;
import java.util.Map;
-@Api(order=100000)
+@Api(order=10000)
@Path("/settings")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
diff --git a/server-core/src/main/java/io/onedev/server/rest/resource/VersionResource.java b/server-core/src/main/java/io/onedev/server/rest/resource/VersionResource.java
new file mode 100644
index 0000000000..141db18541
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/rest/resource/VersionResource.java
@@ -0,0 +1,56 @@
+package io.onedev.server.rest.resource;
+
+import io.onedev.server.OneDev;
+import io.onedev.server.rest.annotation.Api;
+
+import javax.inject.Singleton;
+import javax.ws.rs.*;
+import javax.ws.rs.core.MediaType;
+import java.io.Serializable;
+
+@Api(order=100000, description = "Version info for server and various tools")
+@Path("/version")
+@Produces(MediaType.APPLICATION_JSON)
+@Singleton
+public class VersionResource {
+
+ @Api(order=100, description = "Get server version")
+ @Path("/server")
+ @GET
+ public String getServerVersion() {
+ return OneDev.getInstance().getVersion();
+ }
+
+ @Api(description = "Get tod version range compatible with this server")
+ @Path("/compatible-tod-versions")
+ @GET
+ public VersionRange getCompatibleTodVersions() {
+ return new VersionRange();
+ }
+
+ private static class VersionRange implements Serializable {
+
+ @Api(example = "1.0.0", description = "Minimum version. Null means no lower bound")
+ String minVersion;
+
+ @Api(example = "2.0.0", description = "Maximum version. Null means no upper bound")
+ String maxVersion;
+
+ public String getMinVersion() {
+ return minVersion;
+ }
+
+ public void setMinVersion(String minVersion) {
+ this.minVersion = minVersion;
+ }
+
+ public String getMaxVersion() {
+ return maxVersion;
+ }
+
+ public void setMaxVersion(String maxVersion) {
+ this.maxVersion = maxVersion;
+ }
+ }
+
+}
diff --git a/server-core/src/main/java/io/onedev/server/web/component/build/log/BuildLogPanel.java b/server-core/src/main/java/io/onedev/server/web/component/build/log/BuildLogPanel.java
index 667af75661..e7937d3810 100644
--- a/server-core/src/main/java/io/onedev/server/web/component/build/log/BuildLogPanel.java
+++ b/server-core/src/main/java/io/onedev/server/web/component/build/log/BuildLogPanel.java
@@ -1,36 +1,32 @@
package io.onedev.server.web.component.build.log;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import io.onedev.server.web.page.base.BasePage;
-import org.apache.wicket.ajax.AjaxRequestTarget;
-import org.apache.wicket.core.request.handler.IPartialPageRequestHandler;
-import org.apache.wicket.markup.head.IHeaderResponse;
-import org.apache.wicket.markup.head.JavaScriptHeaderItem;
-import org.apache.wicket.markup.head.OnDomReadyHeaderItem;
-import org.apache.wicket.markup.html.panel.GenericPanel;
-import org.apache.wicket.model.IModel;
-
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets;
-
import io.onedev.server.OneDev;
import io.onedev.server.buildspec.job.log.JobLogEntryEx;
-import io.onedev.server.buildspec.job.log.Message;
import io.onedev.server.job.JobManager;
import io.onedev.server.job.log.LogManager;
import io.onedev.server.job.log.LogSnippet;
import io.onedev.server.model.Build;
import io.onedev.server.model.Build.Status;
import io.onedev.server.security.SecurityUtils;
-import io.onedev.server.web.asset.emoji.Emojis;
import io.onedev.server.web.behavior.AbstractPostAjaxBehavior;
import io.onedev.server.web.behavior.ChangeObserver;
+import io.onedev.server.web.page.base.BasePage;
+import org.apache.wicket.ajax.AjaxRequestTarget;
+import org.apache.wicket.core.request.handler.IPartialPageRequestHandler;
+import org.apache.wicket.markup.head.IHeaderResponse;
+import org.apache.wicket.markup.head.JavaScriptHeaderItem;
+import org.apache.wicket.markup.head.OnDomReadyHeaderItem;
+import org.apache.wicket.markup.html.panel.GenericPanel;
+import org.apache.wicket.model.IModel;
+
+import java.util.Collection;
+import java.util.List;
import static io.onedev.server.model.Build.getDetailChangeObservable;
+import static java.util.stream.Collectors.toList;
@SuppressWarnings("serial")
public class BuildLogPanel extends GenericPanel<Build> {
@@ -52,7 +48,8 @@ public class BuildLogPanel extends GenericPanel<Build> {
add(new ChangeObserver() {
private void appendRecentLogEntries(IPartialPageRequestHandler handler) {
- List<JobLogEntryEx> logEntries = getLogManager().readLogEntries(getBuild(), nextOffset, 0);
+ List<JobLogEntryEx> logEntries = getLogManager().readLogEntries(
+ getBuild().getProject().getId(), getBuild().getNumber(), nextOffset, 0);
if (!logEntries.isEmpty()) {
nextOffset += logEntries.size();
@@ -113,16 +110,9 @@ public class BuildLogPanel extends GenericPanel<Build> {
}
private String asJSON(List<JobLogEntryEx> entries) {
- List<JobLogEntryEx> transformed = new ArrayList<>();
- Emojis emojis = Emojis.getInstance();
- for (JobLogEntryEx entry: entries) {
- List<Message> messages = new ArrayList<>();
- for (Message message: entry.getMessages())
- messages.add(new Message(message.getStyle(), emojis.apply(message.getText())));
- transformed.add(new JobLogEntryEx(entry.getDate(), messages));
- }
+ entries = entries.stream().map(JobLogEntryEx::transformEmojis).collect(toList());
try {
- return OneDev.getInstance(ObjectMapper.class).writeValueAsString(transformed);
+ return OneDev.getInstance(ObjectMapper.class).writeValueAsString(entries);
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
@@ -134,7 +124,8 @@ public class BuildLogPanel extends GenericPanel<Build> {
response.render(JavaScriptHeaderItem.forReference(new BuildLogResourceReference()));
- LogSnippet snippet = getLogManager().readLogSnippetReversely(getBuild(), MAX_LOG_ENTRIES+1);
+ LogSnippet snippet = getLogManager().readLogSnippetReversely(
+ getBuild().getProject().getId(), getBuild().getNumber(), MAX_LOG_ENTRIES+1);
nextOffset = snippet.offset + snippet.entries.size();
diff --git a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java
index aa19d9c3cc..5b2ea791e9 100644
--- a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java
+++ b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java
@@ -83,94 +83,86 @@ public class KubernetesResource {
@Path("/run-server-step")
@Consumes(MediaType.APPLICATION_OCTET_STREAM)
@POST
- public Response runServerStep(@QueryParam("jobToken") String jobToken, InputStream is) {
+ public StreamingOutput runServerStep(@QueryParam("jobToken") String jobToken, InputStream is) {
JobContext jobContext = jobManager.getJobContext(jobToken, true);
- // Make sure we are not occupying a database connection here as we will occupy
- // database connection when running step at project server side
- sessionManager.closeSession();
- try {
- StreamingOutput os = output -> {
- File filesDir = FileUtils.createTempDir();
- try {
- int length = readInt(is);
- List<Integer> stepPosition = new ArrayList<>();
- for (int i=0; i<length; i++)
- stepPosition.add(readInt(is));
-
- Map<String, String> placeholderValues = new HashMap<>();
- length = readInt(is);
- for (int i=0; i<length; i++)
- placeholderValues.put(readString(is), readString(is));
-
- TarUtils.untar(is, filesDir, false);
-
- Map<String, byte[]> outputFiles = jobManager.runServerStep(jobContext,
- stepPosition, filesDir, placeholderValues, true, new TaskLogger() {
+ return os -> {
+ File filesDir = FileUtils.createTempDir();
+ // Make sure we are not occupying a database connection here as we will occupy
+ // database connection when running step at project server side
+ sessionManager.closeSession();
+ try {
+ int length = readInt(is);
+ List<Integer> stepPosition = new ArrayList<>();
+ for (int i=0; i<length; i++)
+ stepPosition.add(readInt(is));
+
+ Map<String, String> placeholderValues = new HashMap<>();
+ length = readInt(is);
+ for (int i=0; i<length; i++)
+ placeholderValues.put(readString(is), readString(is));
+
+ TarUtils.untar(is, filesDir, false);
+
+ Map<String, byte[]> outputFiles = jobManager.runServerStep(jobContext,
+ stepPosition, filesDir, placeholderValues, true, new TaskLogger() {
- @Override
- public void log(String message, String sessionId) {
- // While testing, ngrok.io buffers response and build can not get log entries
- // timely. This won't happen on pagekite however
- KubernetesHelper.writeInt(output, 1);
- KubernetesHelper.writeString(output, message);
- try {
- output.flush();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
+ @Override
+ public void log(String message, String sessionId) {
+ // While testing, ngrok.io buffers response and build can not get log entries
+ // timely. This won't happen on pagekite however
+ KubernetesHelper.writeInt(os, 1);
+ KubernetesHelper.writeString(os, message);
+ try {
+ os.flush();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
}
-
- });
- if (outputFiles == null)
- outputFiles = new HashMap<>();
- byte[] bytes = SerializationUtils.serialize((Serializable) outputFiles);
- KubernetesHelper.writeInt(output, 2);
- KubernetesHelper.writeInt(output, bytes.length);
- output.write(bytes);
- } finally {
- FileUtils.deleteDir(filesDir);
- }
- };
- return Response.ok(os).build();
- } finally {
- sessionManager.openSession();
- }
+ }
+
+ });
+ if (outputFiles == null)
+ outputFiles = new HashMap<>();
+ byte[] bytes = SerializationUtils.serialize((Serializable) outputFiles);
+ KubernetesHelper.writeInt(os, 2);
+ KubernetesHelper.writeInt(os, bytes.length);
+ os.write(bytes);
+ } finally {
+ sessionManager.openSession();
+ FileUtils.deleteDir(filesDir);
+ }
+ };
}
@Path("/download-dependencies")
@Produces(MediaType.APPLICATION_OCTET_STREAM)
@GET
- public Response downloadDependencies(@QueryParam("jobToken") String jobToken) {
- sessionManager.closeSession();
- try {
- StreamingOutput output = os -> {
- JobContext jobContext = jobManager.getJobContext(jobToken, true);
- File tempDir = FileUtils.createTempDir();
- try {
- jobManager.copyDependencies(jobContext, tempDir);
- TarUtils.tar(tempDir, os, false);
- os.flush();
- } finally {
- FileUtils.deleteDir(tempDir);
- }
- };
- return Response.ok(output).build();
- } finally {
- sessionManager.openSession();
- }
+ public StreamingOutput downloadDependencies(@QueryParam("jobToken") String jobToken) {
+ return os -> {
+ JobContext jobContext = jobManager.getJobContext(jobToken, true);
+ File tempDir = FileUtils.createTempDir();
+ sessionManager.closeSession();
+ try {
+ jobManager.copyDependencies(jobContext, tempDir);
+ TarUtils.tar(tempDir, os, false);
+ os.flush();
+ } finally {
+ sessionManager.openSession();
+ FileUtils.deleteDir(tempDir);
+ }
+ };
}
@Path("/download-cache")
@Produces(MediaType.APPLICATION_OCTET_STREAM)
@GET
- public Response downloadCache(
+ public StreamingOutput downloadCache(
@QueryParam("jobToken") String jobToken,
@QueryParam("cacheKey") @Nullable String cacheKey,
@QueryParam("cacheLoadKeys") @Nullable String joinedCacheLoadKeys,
@QueryParam("cachePath") String cachePath) {
- sessionManager.closeSession();
- try {
- StreamingOutput output = os -> {
+ return os -> {
+ sessionManager.closeSession();
+ try {
var jobContext = jobManager.getJobContext(jobToken, true);
if (cacheKey != null) {
jobCacheManager.downloadCache(jobContext.getProjectId(), cacheKey, cachePath, os);
@@ -178,11 +170,10 @@ public class KubernetesResource {
var cacheLoadKeys = Splitter.on('\n').splitToList(joinedCacheLoadKeys);
jobCacheManager.downloadCache(jobContext.getProjectId(), cacheLoadKeys, cachePath, os);
}
- };
- return Response.ok(output).build();
- } finally {
- sessionManager.openSession();
- }
+ } finally {
+ sessionManager.openSession();
+ }
+ };
}
@Path("/upload-cache")