Skip to content

The Unified and Autonomous Development Platform

fix: Pod never becomes ready after upgrading to OneDev 16.7.0

Robin Shen7713d4695dd6
diff --git a/server-core/src/main/java/io/onedev/server/OneDev.java b/server-core/src/main/java/io/onedev/server/OneDev.java
index e946399164..b7420f1a78 100644
--- a/server-core/src/main/java/io/onedev/server/OneDev.java
+++ b/server-core/src/main/java/io/onedev/server/OneDev.java
@@ -115,8 +115,10 @@ public class OneDev extends AbstractPlugin implements Serializable, Runnable {
@Override
public void start() {
var maintenanceFile = getMaintenanceFile(Bootstrap.installDir);
- try (var ignored = MaintenanceProbeServer.start(Bootstrap.installDir)) {
+ try (var ignored = MaintenanceProbeServer.start(Bootstrap.installDir,
+ () -> "not ready: " + initStage.getMessage())) {
while (maintenanceFile.exists()) {
+ initStage = new InitStage("Maintenance in progress");
logger.info("Maintenance in progress, waiting...");
try {
Thread.sleep(5000);
@@ -130,7 +132,9 @@ public class OneDev extends AbstractPlugin implements Serializable, Runnable {
System.setProperty("hsqldb.reconfig_logging", "false");
System.setProperty("hsqldb.method_class_names", "java.lang.Math");
+ starting("Starting cluster (all servers must run the same OneDev version)");
clusterService.start();
+ starting("Initializing database");
sessionFactoryService.start();
taskScheduler.start();
@@ -151,6 +155,7 @@ public class OneDev extends AbstractPlugin implements Serializable, Runnable {
idService.init();
+ starting("Starting services");
sessionService.run(() -> listenerRegistry.post(new SystemStarting()));
}
jettyServiceProvider.get().start();
@@ -326,6 +331,11 @@ public class OneDev extends AbstractPlugin implements Serializable, Runnable {
return initStage;
}
+ private void starting(String message) {
+ initStage = new InitStage(message);
+ logger.info(message);
+ }
+
public boolean isReady() {
return initStage == null;
}
diff --git a/server-core/src/main/java/io/onedev/server/jetty/MaintenanceProbeServer.java b/server-core/src/main/java/io/onedev/server/jetty/MaintenanceProbeServer.java
index f0f69a319d..df3020c4bf 100644
--- a/server-core/src/main/java/io/onedev/server/jetty/MaintenanceProbeServer.java
+++ b/server-core/src/main/java/io/onedev/server/jetty/MaintenanceProbeServer.java
@@ -3,6 +3,7 @@ package io.onedev.server.jetty;
import java.io.File;
import java.net.HttpURLConnection;
import java.net.URL;
+import java.util.function.Supplier;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.Response;
@@ -25,6 +26,10 @@ public class MaintenanceProbeServer implements AutoCloseable {
}
public static MaintenanceProbeServer start(File installDir) {
+ return start(installDir, () -> "not ready: maintenance in progress");
+ }
+
+ public static MaintenanceProbeServer start(File installDir, Supplier<String> notReadyReason) {
var server = new Server();
ServerConfig serverConfig = null;
try {
@@ -35,7 +40,7 @@ public class MaintenanceProbeServer implements AutoCloseable {
connector.setHost(serverConfig.getHttpHost());
connector.setPort(serverConfig.getHttpPort());
server.addConnector(connector);
- server.setHandler(new ProbeHandler(() -> false) {
+ server.setHandler(new ProbeHandler(() -> false, notReadyReason) {
@Override
public boolean handle(Request request, Response response, Callback callback) {
// Agents may connect before the main server takes over this port.
diff --git a/server-core/src/main/java/io/onedev/server/jetty/ProbeHandler.java b/server-core/src/main/java/io/onedev/server/jetty/ProbeHandler.java
index 81739fe88a..4dc49a6109 100644
--- a/server-core/src/main/java/io/onedev/server/jetty/ProbeHandler.java
+++ b/server-core/src/main/java/io/onedev/server/jetty/ProbeHandler.java
@@ -3,6 +3,7 @@ package io.onedev.server.jetty;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.function.BooleanSupplier;
+import java.util.function.Supplier;
import jakarta.servlet.http.HttpServletResponse;
@@ -19,8 +20,15 @@ public class ProbeHandler extends Handler.Abstract {
private final BooleanSupplier readiness;
+ private final Supplier<String> notReadyReason;
+
public ProbeHandler(BooleanSupplier readiness) {
+ this(readiness, () -> "not ready");
+ }
+
+ public ProbeHandler(BooleanSupplier readiness, Supplier<String> notReadyReason) {
this.readiness = readiness;
+ this.notReadyReason = notReadyReason;
}
@Override
@@ -49,7 +57,7 @@ public class ProbeHandler extends Handler.Abstract {
response.getHeaders().put("Content-Type", "text/plain;charset=UTF-8");
response.getHeaders().put("Cache-Control", "no-store");
String body = status == HttpServletResponse.SC_OK ? "ok\n"
- : status == HttpServletResponse.SC_SERVICE_UNAVAILABLE ? "not ready\n" : "";
+ : status == HttpServletResponse.SC_SERVICE_UNAVAILABLE ? notReadyReason.get() + "\n" : "";
response.write(true, request.getMethod().equals("HEAD") ? null
: ByteBuffer.wrap(body.getBytes(StandardCharsets.UTF_8)), callback);
}
diff --git a/server-core/src/test/java/io/onedev/server/jetty/JakartaJettyIntegrationTest.java b/server-core/src/test/java/io/onedev/server/jetty/JakartaJettyIntegrationTest.java
index 311944d79e..5251eab271 100644
--- a/server-core/src/test/java/io/onedev/server/jetty/JakartaJettyIntegrationTest.java
+++ b/server-core/src/test/java/io/onedev/server/jetty/JakartaJettyIntegrationTest.java
@@ -153,11 +153,28 @@ public class JakartaJettyIntegrationTest {
var server = new Server();
var connector = new LocalConnector(server);
server.addConnector(connector);
- server.setHandler(new ProbeHandler(() -> false));
+ var ready = new java.util.concurrent.atomic.AtomicBoolean();
+ var stage = new java.util.concurrent.atomic.AtomicReference<>("Starting cluster");
+ server.setHandler(new ProbeHandler(ready::get, stage::get));
try {
server.start();
assertTrue(connector.getResponse(request("GET", "/healthz")).contains("200 OK"));
- assertTrue(connector.getResponse(request("GET", "/readyz")).contains("503 Service Unavailable"));
+ for (String message : new String[] {"Starting cluster", "Initializing database", "Starting services"}) {
+ stage.set(message);
+ var response = org.eclipse.jetty.http.HttpTester.parseResponse(
+ connector.getResponse(request("GET", "/readyz")));
+ assertEquals(503, response.getStatus());
+ assertEquals(message + "\n", response.getContent());
+ assertEquals("no-store", response.get("Cache-Control"));
+ }
+ String notReadyHead = connector.getResponse(request("HEAD", "/readyz"));
+ assertTrue(notReadyHead.contains("503 Service Unavailable"));
+ assertEquals("", notReadyHead.substring(notReadyHead.indexOf("\r\n\r\n") + 4));
+ ready.set(true);
+ var response = org.eclipse.jetty.http.HttpTester.parseResponse(
+ connector.getResponse(request("GET", "/readyz")));
+ assertEquals(200, response.getStatus());
+ assertEquals("ok\n", response.getContent());
String head = connector.getResponse(request("HEAD", "/healthz"));
assertFalse(head.substring(head.indexOf("\r\n\r\n") + 4).contains("ok"));
String post = connector.getResponse(request("POST", "/healthz"));
diff --git a/server-product/helm/README.md b/server-product/helm/README.md
index f299001f30..d3eadda57a 100644
--- a/server-product/helm/README.md
+++ b/server-product/helm/README.md
@@ -9,4 +9,26 @@ https://onedev.io
https://docs.onedev.io/installation-guide/deploy-to-k8s
### Chart Values
-https://code.onedev.io/onedev/server/~files/main/server-product/helm/values.yaml
\ No newline at end of file
+https://code.onedev.io/onedev/server/~files/main/server-product/helm/values.yaml
+
+### Upgrading
+
+Follow the [Kubernetes upgrade guide](https://docs.onedev.io/upgrade-guide/deploy-to-k8s)
+and use a chart version containing the upgrade hook.
+
+Image upgrades require downtime. The hook shuts down all OneDev pods in this
+release before Helm applies the new image, then Helm restores `onedev.replicas`.
+Unchanged images skip shutdown. Shut down any other servers sharing the database
+separately, and keep PVC retention on scale-down set to `Retain` (the default).
+Keep the StatefulSet name unchanged and any rolling-update partition at zero.
+
+Configure the hook under `onedev.upgrade` in [values.yaml](values.yaml):
+
+- `stopAllServers`: defaults to `true`; disable only when external tooling stops the servers.
+- `stopTimeoutSeconds`: wait for servers to stop, default 240 seconds. Also allow enough time with Helm's `--timeout`.
+- `image`: override the hook image when using a private registry.
+- `podLabels` / `podAnnotations`: configure the hook pod separately; disable sidecar injection here when needed.
+
+If the hook fails, inspect its Job logs, resolve the cause, and retry. The
+StatefulSet may remain scaled to zero. Rolling back the image does not undo
+database migrations.
diff --git a/server-product/helm/files/stop-servers.py b/server-product/helm/files/stop-servers.py
new file mode 100644
index 0000000000..02110a2d66
--- /dev/null
+++ b/server-product/helm/files/stop-servers.py
@@ -0,0 +1,108 @@
+"""Quiesce an existing OneDev StatefulSet before Helm changes its image."""
+import json
+import os
+import ssl
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+
+def stop_servers(api, namespace, name, image, timeout):
+ base = f"/apis/apps/v1/namespaces/{namespace}/statefulsets/{name}"
+ try:
+ statefulset = api("GET", base)
+ except urllib.error.HTTPError as error:
+ if error.code == 404:
+ raise RuntimeError(
+ f"Refusing to upgrade: StatefulSet {namespace}/{name} was not found; "
+ "cannot verify that existing OneDev servers are stopped. Keep the existing "
+ "StatefulSet name when upgrading (check global.nameOverride and "
+ "global.fullnameOverride).") from error
+ raise
+ retention = statefulset["spec"].get("persistentVolumeClaimRetentionPolicy", {})
+ if retention.get("whenScaled") == "Delete":
+ raise RuntimeError(
+ "Refusing to upgrade: persistentVolumeClaimRetentionPolicy.whenScaled is Delete; "
+ "scaling down could delete OneDev data. Set whenScaled to Retain on the existing "
+ "StatefulSet before retrying.")
+ strategy = statefulset["spec"].get("updateStrategy", {})
+ if strategy.get("rollingUpdate", {}).get("partition", 0) != 0:
+ raise RuntimeError(
+ "Refusing to upgrade: the existing StatefulSet has a nonzero rolling-update "
+ "partition; servers could restart with the old image. Set "
+ "spec.updateStrategy.rollingUpdate.partition to 0 on the existing StatefulSet "
+ "before retrying.")
+ uid = statefulset["metadata"]["uid"]
+ selector = urllib.parse.urlencode({"labelSelector": ",".join(
+ f"{key}={value}" for key, value in statefulset["spec"]["selector"]["matchLabels"].items())})
+ pods_path = f"/api/v1/namespaces/{namespace}/pods"
+
+ def pods():
+ return [pod for pod in api("GET", pods_path + "?" + selector)["items"]
+ if any(owner["uid"] == uid for owner in pod["metadata"].get("ownerReferences", []))]
+
+ def server_image(spec):
+ return next(container["image"] for container in spec["containers"]
+ if container["name"] == "onedevserver")
+
+ existing = pods()
+ if (server_image(statefulset["spec"]["template"]["spec"]) == image
+ and all(server_image(pod["spec"]) == image for pod in existing)):
+ print("OneDev image is unchanged; no upgrade shutdown required", flush=True)
+ return
+
+ print("Stopping all OneDev servers before upgrading the shared database", flush=True)
+ api("PATCH", base + "/scale", {"spec": {"replicas": 0}})
+ deadline = time.monotonic() + timeout
+ deleting = set()
+ while True:
+ existing = pods()
+ current = api("GET", base)
+ if current["spec"]["replicas"] != 0:
+ raise RuntimeError("StatefulSet was scaled up during upgrade; refusing to proceed")
+ if (not existing and current.get("status", {}).get("observedGeneration", 0)
+ >= current["metadata"]["generation"]):
+ print("All OneDev pods have stopped; Helm can now apply the new image", flush=True)
+ return
+ # Older StatefulSet controllers can wait for an unready pod during scale-down.
+ # Delete only this StatefulSet's pods, respecting their normal grace period.
+ for pod in existing:
+ metadata = pod["metadata"]
+ if metadata["uid"] not in deleting and not metadata.get("deletionTimestamp"):
+ try:
+ api("DELETE", pods_path + "/" + metadata["name"],
+ {"apiVersion": "v1", "kind": "DeleteOptions",
+ "preconditions": {"uid": metadata["uid"]}})
+ except urllib.error.HTTPError as error:
+ if error.code != 404:
+ raise
+ deleting.add(metadata["uid"])
+ if time.monotonic() >= deadline:
+ raise TimeoutError("OneDev pods did not stop; upgrade aborted with replicas left at zero")
+ time.sleep(2)
+
+
+def main():
+ credentials = Path("/var/run/secrets/kubernetes.io/serviceaccount")
+ context = ssl.create_default_context(cafile=str(credentials / "ca.crt"))
+ host = os.environ["KUBERNETES_SERVICE_HOST"]
+ if ":" in host:
+ host = f"[{host}]"
+ endpoint = f"https://{host}:{os.environ['KUBERNETES_SERVICE_PORT']}"
+
+ def api(method, path, body=None):
+ request = urllib.request.Request(endpoint + path, method=method,
+ data=json.dumps(body).encode() if body is not None else None,
+ headers={"Authorization": "Bearer " + (credentials / "token").read_text().strip(),
+ "Content-Type": "application/merge-patch+json" if method == "PATCH" else "application/json"})
+ with urllib.request.urlopen(request, context=context, timeout=15) as response:
+ return json.load(response)
+
+ stop_servers(api, os.environ["POD_NAMESPACE"], os.environ["STATEFULSET_NAME"],
+ os.environ["TARGET_IMAGE"], int(os.environ["STOP_TIMEOUT"]))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/server-product/helm/templates/upgrade-hook.yaml b/server-product/helm/templates/upgrade-hook.yaml
new file mode 100644
index 0000000000..f91eb10b5b
--- /dev/null
+++ b/server-product/helm/templates/upgrade-hook.yaml
@@ -0,0 +1,127 @@
+{{- $upgrade := .Values.onedev.upgrade | default dict -}}
+{{- if and .Release.IsUpgrade (dig "stopAllServers" true $upgrade) -}}
+{{- $strategy := .Values.onedev.updateStrategy | default dict -}}
+{{- $rollingUpdate := $strategy.rollingUpdate | default dict -}}
+{{- if ne (int (default 0 $rollingUpdate.partition)) 0 -}}
+{{- fail "onedev.updateStrategy.rollingUpdate.partition must be 0 when onedev.upgrade.stopAllServers is true; a nonzero partition can restart servers with the old image after stopping them" -}}
+{{- end -}}
+{{- $name := printf "%s-upgrade" (include "ods.fullname" . | trunc 55 | trimSuffix "-") -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: {{ $name }}
+ namespace: {{ .Release.Namespace }}
+ annotations:
+ helm.sh/hook: pre-upgrade
+ helm.sh/hook-weight: "-30"
+ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: {{ $name }}
+ namespace: {{ .Release.Namespace }}
+ annotations:
+ helm.sh/hook: pre-upgrade
+ helm.sh/hook-weight: "-20"
+ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+rules:
+ - apiGroups: ["apps"]
+ resources: ["statefulsets"]
+ resourceNames: [{{ include "ods.fullname" . | quote }}]
+ verbs: ["get"]
+ - apiGroups: ["apps"]
+ resources: ["statefulsets/scale"]
+ resourceNames: [{{ include "ods.fullname" . | quote }}]
+ verbs: ["patch"]
+ - apiGroups: [""]
+ resources: ["pods"]
+ verbs: ["list", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: {{ $name }}
+ namespace: {{ .Release.Namespace }}
+ annotations:
+ helm.sh/hook: pre-upgrade
+ helm.sh/hook-weight: "-10"
+ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+subjects:
+ - kind: ServiceAccount
+ name: {{ $name }}
+ namespace: {{ .Release.Namespace }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: {{ $name }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: {{ $name }}
+ namespace: {{ .Release.Namespace }}
+ annotations:
+ helm.sh/hook: pre-upgrade
+ helm.sh/hook-weight: "0"
+ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+spec:
+ backoffLimit: 0
+ activeDeadlineSeconds: {{ add (int (default 240 $upgrade.stopTimeoutSeconds)) 30 }}
+ template:
+{{- if or $upgrade.podLabels $upgrade.podAnnotations }}
+ metadata:
+{{- with $upgrade.podLabels }}
+ labels: {{ toYaml . | nindent 8 }}
+{{- end }}
+{{- with $upgrade.podAnnotations }}
+ annotations: {{ toYaml . | nindent 8 }}
+{{- end }}
+{{- end }}
+ spec:
+ serviceAccountName: {{ $name }}
+ restartPolicy: Never
+ securityContext:
+ seccompProfile:
+ type: RuntimeDefault
+{{- if .Values.image.pullSecrets }}
+ imagePullSecrets: {{ toYaml .Values.image.pullSecrets | nindent 8 }}
+{{- end }}
+{{- if .Values.nodeSelector }}
+ nodeSelector: {{ toYaml .Values.nodeSelector | nindent 8 }}
+{{- end }}
+{{- if .Values.tolerations }}
+ tolerations: {{ toYaml .Values.tolerations | nindent 8 }}
+{{- end }}
+ containers:
+ - name: stop-servers
+ image: {{ default "docker.io/library/python:3.13-alpine" $upgrade.image | quote }}
+ command: ["python3", "-c"]
+ args:
+ - |
+{{ .Files.Get "files/stop-servers.py" | indent 14 }}
+ env:
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: STATEFULSET_NAME
+ value: {{ include "ods.fullname" . | quote }}
+ - name: TARGET_IMAGE
+ value: "{{ .Values.image.repository }}/{{ .Values.image.name }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+ - name: STOP_TIMEOUT
+ value: {{ default 240 $upgrade.stopTimeoutSeconds | quote }}
+ securityContext:
+ allowPrivilegeEscalation: false
+ readOnlyRootFilesystem: true
+ runAsNonRoot: true
+ runAsUser: 65534
+ capabilities:
+ drop: ["ALL"]
+ resources:
+ requests:
+ cpu: 10m
+ memory: 32Mi
+ limits:
+ memory: 64Mi
+{{- end }}
diff --git a/server-product/helm/values.yaml b/server-product/helm/values.yaml
index 3cf2d137f6..24ed95ea3a 100644
--- a/server-product/helm/values.yaml
+++ b/server-product/helm/values.yaml
@@ -17,6 +17,26 @@ onedev:
# onedev.updateStrategy -- Valid options: `RollingUpdate`, `OnDelete`
updateStrategy:
type: RollingUpdate
+ upgrade:
+ # onedev.upgrade.stopAllServers -- Stop all OneDev servers in this release before Helm applies a
+ # new image, then restore the configured replica count. Required for shared database migrations
+ # and incompatible cluster versions. Image upgrades require downtime; unchanged images skip shutdown.
+ # Set false only if external deployment tooling coordinates shutdown. This does not disable upgrades.
+ stopAllServers: true
+ # onedev.upgrade.image -- Python 3 image used by the pre-upgrade hook. Override for a private mirror.
+ image: docker.io/library/python:3.13-alpine
+ # onedev.upgrade.stopTimeoutSeconds -- Maximum time to wait for old OneDev pods to terminate
+ # after requesting shutdown. The hook Job's overall deadline is this value plus 30 seconds,
+ # including scheduling and image pulling. Set helm --timeout separately to allow for the hook
+ # and application startup.
+ stopTimeoutSeconds: 240
+ # onedev.upgrade.podLabels -- Labels for the upgrade hook pod, independent of application pod labels.
+ # Use these to disable sidecar injection when needed so the hook Job can finish.
+ podLabels: {}
+ # sidecar.istio.io/inject: "false"
+ # onedev.upgrade.podAnnotations -- Annotations for the upgrade hook pod, independent of application
+ # pod annotations. Set annotation-based sidecar injection controls here when needed.
+ podAnnotations: {}
trustCerts:
# onedev.trustCerts.secretName -- Name of an existing secret containing trusted certificates. You may create the
# secret with "kubectl create secret generic onedev-trustcerts -n onedev --from-file=/path/to/trust-certs",
@@ -326,10 +346,26 @@ dnsConfig: {}
# livenessProbe -- Specify the livenessProbe
# [configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes).
-# OneDev provides `/healthz`; allow for a brief outage while it switches to maintenance mode.
-livenessProbe: {}
+# Allow startup time and brief outages when switching between application and maintenance servers.
+# Set to null to disable.
+livenessProbe:
+ httpGet:
+ path: /healthz
+ port: http
+ initialDelaySeconds: 300
+ periodSeconds: 10
+ timeoutSeconds: 3
+ failureThreshold: 6
# readinessProbe -- Specify the readinessProbe
# [configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes).
# OneDev provides `/readyz` for readiness checks. It returns 503 during startup, shutdown, and maintenance.
-readinessProbe: {}
+# Set to null to disable.
+readinessProbe:
+ httpGet:
+ path: /readyz
+ port: http
+ initialDelaySeconds: 5
+ periodSeconds: 5
+ timeoutSeconds: 3
+ failureThreshold: 3