The Unified and Autonomous Development Platform
fix: Pod never becomes ready after upgrading to OneDev 16.7.0
Robin Shen7713d4695dd6
diff --git a/server-core/src/main/java/io/onedev/server/OneDev.java b/server-core/src/main/java/io/onedev/server/OneDev.javaindex e946399164..b7420f1a78 100644--- a/server-core/src/main/java/io/onedev/server/OneDev.java+++ b/server-core/src/main/java/io/onedev/server/OneDev.java@@ -115,8 +115,10 @@ public class OneDev extends AbstractPlugin implements Serializable, Runnable {@Overridepublic void start() {var maintenanceFile = getMaintenanceFile(Bootstrap.installDir);- try (var ignored = MaintenanceProbeServer.start(Bootstrap.installDir)) {+ try (var ignored = MaintenanceProbeServer.start(Bootstrap.installDir,+ () -> "not ready: " + initStage.getMessage())) {while (maintenanceFile.exists()) {+ initStage = new InitStage("Maintenance in progress");logger.info("Maintenance in progress, waiting...");try {Thread.sleep(5000);@@ -130,7 +132,9 @@ public class OneDev extends AbstractPlugin implements Serializable, Runnable {System.setProperty("hsqldb.reconfig_logging", "false");System.setProperty("hsqldb.method_class_names", "java.lang.Math");+ starting("Starting cluster (all servers must run the same OneDev version)");clusterService.start();+ starting("Initializing database");sessionFactoryService.start();taskScheduler.start();@@ -151,6 +155,7 @@ public class OneDev extends AbstractPlugin implements Serializable, Runnable {idService.init();+ starting("Starting services");sessionService.run(() -> listenerRegistry.post(new SystemStarting()));}jettyServiceProvider.get().start();@@ -326,6 +331,11 @@ public class OneDev extends AbstractPlugin implements Serializable, Runnable {return initStage;}+ private void starting(String message) {+ initStage = new InitStage(message);+ logger.info(message);+ }+public boolean isReady() {return initStage == null;}diff --git a/server-core/src/main/java/io/onedev/server/jetty/MaintenanceProbeServer.java b/server-core/src/main/java/io/onedev/server/jetty/MaintenanceProbeServer.javaindex f0f69a319d..df3020c4bf 100644--- a/server-core/src/main/java/io/onedev/server/jetty/MaintenanceProbeServer.java+++ b/server-core/src/main/java/io/onedev/server/jetty/MaintenanceProbeServer.java@@ -3,6 +3,7 @@ package io.onedev.server.jetty;import java.io.File;import java.net.HttpURLConnection;import java.net.URL;+import java.util.function.Supplier;import org.eclipse.jetty.server.Request;import org.eclipse.jetty.server.Response;@@ -25,6 +26,10 @@ public class MaintenanceProbeServer implements AutoCloseable {}public static MaintenanceProbeServer start(File installDir) {+ return start(installDir, () -> "not ready: maintenance in progress");+ }++ public static MaintenanceProbeServer start(File installDir, Supplier<String> notReadyReason) {var server = new Server();ServerConfig serverConfig = null;try {@@ -35,7 +40,7 @@ public class MaintenanceProbeServer implements AutoCloseable {connector.setHost(serverConfig.getHttpHost());connector.setPort(serverConfig.getHttpPort());server.addConnector(connector);- server.setHandler(new ProbeHandler(() -> false) {+ server.setHandler(new ProbeHandler(() -> false, notReadyReason) {@Overridepublic boolean handle(Request request, Response response, Callback callback) {// Agents may connect before the main server takes over this port.diff --git a/server-core/src/main/java/io/onedev/server/jetty/ProbeHandler.java b/server-core/src/main/java/io/onedev/server/jetty/ProbeHandler.javaindex 81739fe88a..4dc49a6109 100644--- a/server-core/src/main/java/io/onedev/server/jetty/ProbeHandler.java+++ b/server-core/src/main/java/io/onedev/server/jetty/ProbeHandler.java@@ -3,6 +3,7 @@ package io.onedev.server.jetty;import java.nio.ByteBuffer;import java.nio.charset.StandardCharsets;import java.util.function.BooleanSupplier;+import java.util.function.Supplier;import jakarta.servlet.http.HttpServletResponse;@@ -19,8 +20,15 @@ public class ProbeHandler extends Handler.Abstract {private final BooleanSupplier readiness;+ private final Supplier<String> notReadyReason;+public ProbeHandler(BooleanSupplier readiness) {+ this(readiness, () -> "not ready");+ }++ public ProbeHandler(BooleanSupplier readiness, Supplier<String> notReadyReason) {this.readiness = readiness;+ this.notReadyReason = notReadyReason;}@Override@@ -49,7 +57,7 @@ public class ProbeHandler extends Handler.Abstract {response.getHeaders().put("Content-Type", "text/plain;charset=UTF-8");response.getHeaders().put("Cache-Control", "no-store");String body = status == HttpServletResponse.SC_OK ? "ok\n"- : status == HttpServletResponse.SC_SERVICE_UNAVAILABLE ? "not ready\n" : "";+ : status == HttpServletResponse.SC_SERVICE_UNAVAILABLE ? notReadyReason.get() + "\n" : "";response.write(true, request.getMethod().equals("HEAD") ? null: ByteBuffer.wrap(body.getBytes(StandardCharsets.UTF_8)), callback);}diff --git a/server-core/src/test/java/io/onedev/server/jetty/JakartaJettyIntegrationTest.java b/server-core/src/test/java/io/onedev/server/jetty/JakartaJettyIntegrationTest.javaindex 311944d79e..5251eab271 100644--- a/server-core/src/test/java/io/onedev/server/jetty/JakartaJettyIntegrationTest.java+++ b/server-core/src/test/java/io/onedev/server/jetty/JakartaJettyIntegrationTest.java@@ -153,11 +153,28 @@ public class JakartaJettyIntegrationTest {var server = new Server();var connector = new LocalConnector(server);server.addConnector(connector);- server.setHandler(new ProbeHandler(() -> false));+ var ready = new java.util.concurrent.atomic.AtomicBoolean();+ var stage = new java.util.concurrent.atomic.AtomicReference<>("Starting cluster");+ server.setHandler(new ProbeHandler(ready::get, stage::get));try {server.start();assertTrue(connector.getResponse(request("GET", "/healthz")).contains("200 OK"));- assertTrue(connector.getResponse(request("GET", "/readyz")).contains("503 Service Unavailable"));+ for (String message : new String[] {"Starting cluster", "Initializing database", "Starting services"}) {+ stage.set(message);+ var response = org.eclipse.jetty.http.HttpTester.parseResponse(+ connector.getResponse(request("GET", "/readyz")));+ assertEquals(503, response.getStatus());+ assertEquals(message + "\n", response.getContent());+ assertEquals("no-store", response.get("Cache-Control"));+ }+ String notReadyHead = connector.getResponse(request("HEAD", "/readyz"));+ assertTrue(notReadyHead.contains("503 Service Unavailable"));+ assertEquals("", notReadyHead.substring(notReadyHead.indexOf("\r\n\r\n") + 4));+ ready.set(true);+ var response = org.eclipse.jetty.http.HttpTester.parseResponse(+ connector.getResponse(request("GET", "/readyz")));+ assertEquals(200, response.getStatus());+ assertEquals("ok\n", response.getContent());String head = connector.getResponse(request("HEAD", "/healthz"));assertFalse(head.substring(head.indexOf("\r\n\r\n") + 4).contains("ok"));String post = connector.getResponse(request("POST", "/healthz"));diff --git a/server-product/helm/README.md b/server-product/helm/README.mdindex f299001f30..d3eadda57a 100644--- a/server-product/helm/README.md+++ b/server-product/helm/README.md@@ -9,4 +9,26 @@ https://onedev.iohttps://docs.onedev.io/installation-guide/deploy-to-k8s### Chart Values-https://code.onedev.io/onedev/server/~files/main/server-product/helm/values.yaml\ No newline at end of file+https://code.onedev.io/onedev/server/~files/main/server-product/helm/values.yaml++### Upgrading++Follow the [Kubernetes upgrade guide](https://docs.onedev.io/upgrade-guide/deploy-to-k8s)+and use a chart version containing the upgrade hook.++Image upgrades require downtime. The hook shuts down all OneDev pods in this+release before Helm applies the new image, then Helm restores `onedev.replicas`.+Unchanged images skip shutdown. Shut down any other servers sharing the database+separately, and keep PVC retention on scale-down set to `Retain` (the default).+Keep the StatefulSet name unchanged and any rolling-update partition at zero.++Configure the hook under `onedev.upgrade` in [values.yaml](values.yaml):++- `stopAllServers`: defaults to `true`; disable only when external tooling stops the servers.+- `stopTimeoutSeconds`: wait for servers to stop, default 240 seconds. Also allow enough time with Helm's `--timeout`.+- `image`: override the hook image when using a private registry.+- `podLabels` / `podAnnotations`: configure the hook pod separately; disable sidecar injection here when needed.++If the hook fails, inspect its Job logs, resolve the cause, and retry. The+StatefulSet may remain scaled to zero. Rolling back the image does not undo+database migrations.diff --git a/server-product/helm/files/stop-servers.py b/server-product/helm/files/stop-servers.pynew file mode 100644index 0000000000..02110a2d66--- /dev/null+++ b/server-product/helm/files/stop-servers.py@@ -0,0 +1,108 @@+"""Quiesce an existing OneDev StatefulSet before Helm changes its image."""+import json+import os+import ssl+import time+import urllib.error+import urllib.parse+import urllib.request+from pathlib import Path+++def stop_servers(api, namespace, name, image, timeout):+ base = f"/apis/apps/v1/namespaces/{namespace}/statefulsets/{name}"+ try:+ statefulset = api("GET", base)+ except urllib.error.HTTPError as error:+ if error.code == 404:+ raise RuntimeError(+ f"Refusing to upgrade: StatefulSet {namespace}/{name} was not found; "+ "cannot verify that existing OneDev servers are stopped. Keep the existing "+ "StatefulSet name when upgrading (check global.nameOverride and "+ "global.fullnameOverride).") from error+ raise+ retention = statefulset["spec"].get("persistentVolumeClaimRetentionPolicy", {})+ if retention.get("whenScaled") == "Delete":+ raise RuntimeError(+ "Refusing to upgrade: persistentVolumeClaimRetentionPolicy.whenScaled is Delete; "+ "scaling down could delete OneDev data. Set whenScaled to Retain on the existing "+ "StatefulSet before retrying.")+ strategy = statefulset["spec"].get("updateStrategy", {})+ if strategy.get("rollingUpdate", {}).get("partition", 0) != 0:+ raise RuntimeError(+ "Refusing to upgrade: the existing StatefulSet has a nonzero rolling-update "+ "partition; servers could restart with the old image. Set "+ "spec.updateStrategy.rollingUpdate.partition to 0 on the existing StatefulSet "+ "before retrying.")+ uid = statefulset["metadata"]["uid"]+ selector = urllib.parse.urlencode({"labelSelector": ",".join(+ f"{key}={value}" for key, value in statefulset["spec"]["selector"]["matchLabels"].items())})+ pods_path = f"/api/v1/namespaces/{namespace}/pods"++ def pods():+ return [pod for pod in api("GET", pods_path + "?" + selector)["items"]+ if any(owner["uid"] == uid for owner in pod["metadata"].get("ownerReferences", []))]++ def server_image(spec):+ return next(container["image"] for container in spec["containers"]+ if container["name"] == "onedevserver")++ existing = pods()+ if (server_image(statefulset["spec"]["template"]["spec"]) == image+ and all(server_image(pod["spec"]) == image for pod in existing)):+ print("OneDev image is unchanged; no upgrade shutdown required", flush=True)+ return++ print("Stopping all OneDev servers before upgrading the shared database", flush=True)+ api("PATCH", base + "/scale", {"spec": {"replicas": 0}})+ deadline = time.monotonic() + timeout+ deleting = set()+ while True:+ existing = pods()+ current = api("GET", base)+ if current["spec"]["replicas"] != 0:+ raise RuntimeError("StatefulSet was scaled up during upgrade; refusing to proceed")+ if (not existing and current.get("status", {}).get("observedGeneration", 0)+ >= current["metadata"]["generation"]):+ print("All OneDev pods have stopped; Helm can now apply the new image", flush=True)+ return+ # Older StatefulSet controllers can wait for an unready pod during scale-down.+ # Delete only this StatefulSet's pods, respecting their normal grace period.+ for pod in existing:+ metadata = pod["metadata"]+ if metadata["uid"] not in deleting and not metadata.get("deletionTimestamp"):+ try:+ api("DELETE", pods_path + "/" + metadata["name"],+ {"apiVersion": "v1", "kind": "DeleteOptions",+ "preconditions": {"uid": metadata["uid"]}})+ except urllib.error.HTTPError as error:+ if error.code != 404:+ raise+ deleting.add(metadata["uid"])+ if time.monotonic() >= deadline:+ raise TimeoutError("OneDev pods did not stop; upgrade aborted with replicas left at zero")+ time.sleep(2)+++def main():+ credentials = Path("/var/run/secrets/kubernetes.io/serviceaccount")+ context = ssl.create_default_context(cafile=str(credentials / "ca.crt"))+ host = os.environ["KUBERNETES_SERVICE_HOST"]+ if ":" in host:+ host = f"[{host}]"+ endpoint = f"https://{host}:{os.environ['KUBERNETES_SERVICE_PORT']}"++ def api(method, path, body=None):+ request = urllib.request.Request(endpoint + path, method=method,+ data=json.dumps(body).encode() if body is not None else None,+ headers={"Authorization": "Bearer " + (credentials / "token").read_text().strip(),+ "Content-Type": "application/merge-patch+json" if method == "PATCH" else "application/json"})+ with urllib.request.urlopen(request, context=context, timeout=15) as response:+ return json.load(response)++ stop_servers(api, os.environ["POD_NAMESPACE"], os.environ["STATEFULSET_NAME"],+ os.environ["TARGET_IMAGE"], int(os.environ["STOP_TIMEOUT"]))+++if __name__ == "__main__":+ main()diff --git a/server-product/helm/templates/upgrade-hook.yaml b/server-product/helm/templates/upgrade-hook.yamlnew file mode 100644index 0000000000..f91eb10b5b--- /dev/null+++ b/server-product/helm/templates/upgrade-hook.yaml@@ -0,0 +1,127 @@+{{- $upgrade := .Values.onedev.upgrade | default dict -}}+{{- if and .Release.IsUpgrade (dig "stopAllServers" true $upgrade) -}}+{{- $strategy := .Values.onedev.updateStrategy | default dict -}}+{{- $rollingUpdate := $strategy.rollingUpdate | default dict -}}+{{- if ne (int (default 0 $rollingUpdate.partition)) 0 -}}+{{- fail "onedev.updateStrategy.rollingUpdate.partition must be 0 when onedev.upgrade.stopAllServers is true; a nonzero partition can restart servers with the old image after stopping them" -}}+{{- end -}}+{{- $name := printf "%s-upgrade" (include "ods.fullname" . | trunc 55 | trimSuffix "-") -}}+apiVersion: v1+kind: ServiceAccount+metadata:+ name: {{ $name }}+ namespace: {{ .Release.Namespace }}+ annotations:+ helm.sh/hook: pre-upgrade+ helm.sh/hook-weight: "-30"+ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded+---+apiVersion: rbac.authorization.k8s.io/v1+kind: Role+metadata:+ name: {{ $name }}+ namespace: {{ .Release.Namespace }}+ annotations:+ helm.sh/hook: pre-upgrade+ helm.sh/hook-weight: "-20"+ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded+rules:+ - apiGroups: ["apps"]+ resources: ["statefulsets"]+ resourceNames: [{{ include "ods.fullname" . | quote }}]+ verbs: ["get"]+ - apiGroups: ["apps"]+ resources: ["statefulsets/scale"]+ resourceNames: [{{ include "ods.fullname" . | quote }}]+ verbs: ["patch"]+ - apiGroups: [""]+ resources: ["pods"]+ verbs: ["list", "delete"]+---+apiVersion: rbac.authorization.k8s.io/v1+kind: RoleBinding+metadata:+ name: {{ $name }}+ namespace: {{ .Release.Namespace }}+ annotations:+ helm.sh/hook: pre-upgrade+ helm.sh/hook-weight: "-10"+ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded+subjects:+ - kind: ServiceAccount+ name: {{ $name }}+ namespace: {{ .Release.Namespace }}+roleRef:+ apiGroup: rbac.authorization.k8s.io+ kind: Role+ name: {{ $name }}+---+apiVersion: batch/v1+kind: Job+metadata:+ name: {{ $name }}+ namespace: {{ .Release.Namespace }}+ annotations:+ helm.sh/hook: pre-upgrade+ helm.sh/hook-weight: "0"+ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded+spec:+ backoffLimit: 0+ activeDeadlineSeconds: {{ add (int (default 240 $upgrade.stopTimeoutSeconds)) 30 }}+ template:+{{- if or $upgrade.podLabels $upgrade.podAnnotations }}+ metadata:+{{- with $upgrade.podLabels }}+ labels: {{ toYaml . | nindent 8 }}+{{- end }}+{{- with $upgrade.podAnnotations }}+ annotations: {{ toYaml . | nindent 8 }}+{{- end }}+{{- end }}+ spec:+ serviceAccountName: {{ $name }}+ restartPolicy: Never+ securityContext:+ seccompProfile:+ type: RuntimeDefault+{{- if .Values.image.pullSecrets }}+ imagePullSecrets: {{ toYaml .Values.image.pullSecrets | nindent 8 }}+{{- end }}+{{- if .Values.nodeSelector }}+ nodeSelector: {{ toYaml .Values.nodeSelector | nindent 8 }}+{{- end }}+{{- if .Values.tolerations }}+ tolerations: {{ toYaml .Values.tolerations | nindent 8 }}+{{- end }}+ containers:+ - name: stop-servers+ image: {{ default "docker.io/library/python:3.13-alpine" $upgrade.image | quote }}+ command: ["python3", "-c"]+ args:+ - |+{{ .Files.Get "files/stop-servers.py" | indent 14 }}+ env:+ - name: POD_NAMESPACE+ valueFrom:+ fieldRef:+ fieldPath: metadata.namespace+ - name: STATEFULSET_NAME+ value: {{ include "ods.fullname" . | quote }}+ - name: TARGET_IMAGE+ value: "{{ .Values.image.repository }}/{{ .Values.image.name }}:{{ .Values.image.tag | default .Chart.AppVersion }}"+ - name: STOP_TIMEOUT+ value: {{ default 240 $upgrade.stopTimeoutSeconds | quote }}+ securityContext:+ allowPrivilegeEscalation: false+ readOnlyRootFilesystem: true+ runAsNonRoot: true+ runAsUser: 65534+ capabilities:+ drop: ["ALL"]+ resources:+ requests:+ cpu: 10m+ memory: 32Mi+ limits:+ memory: 64Mi+{{- end }}diff --git a/server-product/helm/values.yaml b/server-product/helm/values.yamlindex 3cf2d137f6..24ed95ea3a 100644--- a/server-product/helm/values.yaml+++ b/server-product/helm/values.yaml@@ -17,6 +17,26 @@ onedev:# onedev.updateStrategy -- Valid options: `RollingUpdate`, `OnDelete`updateStrategy:type: RollingUpdate+ upgrade:+ # onedev.upgrade.stopAllServers -- Stop all OneDev servers in this release before Helm applies a+ # new image, then restore the configured replica count. Required for shared database migrations+ # and incompatible cluster versions. Image upgrades require downtime; unchanged images skip shutdown.+ # Set false only if external deployment tooling coordinates shutdown. This does not disable upgrades.+ stopAllServers: true+ # onedev.upgrade.image -- Python 3 image used by the pre-upgrade hook. Override for a private mirror.+ image: docker.io/library/python:3.13-alpine+ # onedev.upgrade.stopTimeoutSeconds -- Maximum time to wait for old OneDev pods to terminate+ # after requesting shutdown. The hook Job's overall deadline is this value plus 30 seconds,+ # including scheduling and image pulling. Set helm --timeout separately to allow for the hook+ # and application startup.+ stopTimeoutSeconds: 240+ # onedev.upgrade.podLabels -- Labels for the upgrade hook pod, independent of application pod labels.+ # Use these to disable sidecar injection when needed so the hook Job can finish.+ podLabels: {}+ # sidecar.istio.io/inject: "false"+ # onedev.upgrade.podAnnotations -- Annotations for the upgrade hook pod, independent of application+ # pod annotations. Set annotation-based sidecar injection controls here when needed.+ podAnnotations: {}trustCerts:# onedev.trustCerts.secretName -- Name of an existing secret containing trusted certificates. You may create the# secret with "kubectl create secret generic onedev-trustcerts -n onedev --from-file=/path/to/trust-certs",@@ -326,10 +346,26 @@ dnsConfig: {}# livenessProbe -- Specify the livenessProbe# [configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes).-# OneDev provides `/healthz`; allow for a brief outage while it switches to maintenance mode.-livenessProbe: {}+# Allow startup time and brief outages when switching between application and maintenance servers.+# Set to null to disable.+livenessProbe:+ httpGet:+ path: /healthz+ port: http+ initialDelaySeconds: 300+ periodSeconds: 10+ timeoutSeconds: 3+ failureThreshold: 6# readinessProbe -- Specify the readinessProbe# [configuration](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes).# OneDev provides `/readyz` for readiness checks. It returns 503 during startup, shutdown, and maintenance.-readinessProbe: {}+# Set to null to disable.+readinessProbe:+ httpGet:+ path: /readyz+ port: http+ initialDelaySeconds: 5+ periodSeconds: 5+ timeoutSeconds: 3+ failureThreshold: 3