Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Option to enable OpenTelemetry agent on manager JVM (#3218)

Eric Bariauxba57e6eab182
diff --git a/README.md b/README.md
index 53cc0d183..51acdb6f9 100644
--- a/README.md
+++ b/README.md
@@ -73,6 +73,10 @@ Persistent data is stored in a PostgreSQL DB which is stored in the `openremote_
Note that historical attribute data is purged daily based on value of `OR_DATA_POINTS_MAX_AGE_DAYS`; this value can also be overridden for individual attributes by using the `dataPointsMaxAgeDays` configuration item.
See the [Developer Guide](https://docs.openremote.io/docs/developer-guide/useful-commands-and-queries/#backuprestore-openremote-db) for details on making backups of the database.
+## OpenTelemetry tracing
+
+The manager distribution and container image support OpenTelemetry tracing. See [Traces](https://docs.openremote.io/docs/user-guide/observability/traces) for more information.
+
## Contributing to OpenRemote
For information and how to set up a development environment, see the [Developer Guide](https://docs.openremote.io/docs/category/developer-guide).
diff --git a/docker-compose.yml b/docker-compose.yml
index a0d38e938..8ded0c321 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -79,6 +79,16 @@ services:
OR_SSL_PORT: ${OR_SSL_PORT:--1}
OR_DEV_MODE: ${OR_DEV_MODE:-false}
+ # Opt-in OpenTelemetry tracing to a local Alloy OTLP/HTTP receiver.
+ OTEL_JAVAAGENT_ENABLED: ${OTEL_JAVAAGENT_ENABLED:-false}
+ OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-openremote-manager}
+ OTEL_TRACES_EXPORTER: ${OTEL_TRACES_EXPORTER:-otlp}
+ OTEL_EXPORTER_OTLP_ENDPOINT:
+ OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
+ OTEL_METRICS_EXPORTER: ${OTEL_METRICS_EXPORTER:-none}
+ OTEL_LOGS_EXPORTER: ${OTEL_LOGS_EXPORTER:-none}
+ OTEL_INSTRUMENTATION_COMMON_DB_STATEMENT_SANITIZER_ENABLED: "true"
+
# The following variables will configure the demo
OR_FORECAST_SOLAR_API_KEY:
OR_OPEN_WEATHER_API_APP_ID:
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index ae2fcf67c..6e04a2889 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -55,6 +55,7 @@ niftyModbus = "1.4.0"
nimbusJoseJwt = "10.9.1"
objenesis = "3.5"
moquette = "0.18.0"
+opentelemetryJavaagent = "2.30.0"
postgresJdbc = "42.7.13"
prometheus = "1.6.1"
protobuf = "3.25.5"
@@ -164,6 +165,7 @@ niftyModbusTcp = { module = "net.solarnetwork.common:nifty-modbus-tcp", version.
nimbusJoseJwt = { module = "com.nimbusds:nimbus-jose-jwt", version.ref = "nimbusJoseJwt" }
objenesis = { module = "org.objenesis:objenesis", version.ref = "objenesis" }
moquette = { module = "com.github.moquette-io:moquette", version.ref = "moquette" }
+opentelemetryJavaagent = { module = "io.opentelemetry.javaagent:opentelemetry-javaagent", version.ref = "opentelemetryJavaagent" }
orZwave = { module = "org.openremote:or-zwave", version.ref = "zwave" }
postgresql = { module = "org.postgresql:postgresql", version.ref = "postgresJdbc" }
prometheusMetricsExporterHttpserver = { module = "io.prometheus:prometheus-metrics-exporter-httpserver", version.ref = "prometheus" }
diff --git a/kubernetes/README.md b/kubernetes/README.md
index ee52dadc0..adbebf8bb 100644
--- a/kubernetes/README.md
+++ b/kubernetes/README.md
@@ -155,6 +155,28 @@ The manager configuration has 2 different flags that related to metrics:
- `service.metrics.enabled` indicating if a metrics service for the manager should be exposed
This is only effective if the manager exposes metrics i.e. both flags must be true for the service to be created.
+### OpenTelemetry tracing
+
+The manager image contains a pinned OpenTelemetry Java agent, but does not attach it unless
+`or.otel.enabled` is explicitly set to `true`. Configure the agent through `or.otel` in an environment-specific
+values file. For example, if Alloy is exposed by the `alloy-otel` service in the `observability` namespace:
+
+```yaml
+or:
+ otel:
+ enabled: true
+ serviceName: openremote-manager
+ endpoint: http://alloy-otel.observability.svc.cluster.local:4318
+ protocol: http/protobuf
+```
+
+Replace the service name and namespace with those of the local Alloy deployment. No Tempo credentials belong in
+the manager configuration: Alloy is the local OTLP endpoint and is responsible for authenticated forwarding.
+Additional OpenTelemetry Java agent settings, such as sampling, can be supplied through `or.env`; environment
+variables managed by `or.otel` must not be duplicated there when tracing is enabled. See
+the [top-level OpenTelemetry tracing documentation](../README.md#opentelemetry-tracing) for validation and data-safety
+guidance.
+
### JMX
The manager can optionally (it is disabled by default) expose a service to provide JMX access.
diff --git a/kubernetes/manager/templates/_helpers.tpl b/kubernetes/manager/templates/_helpers.tpl
index 67034f6d9..0b9927b4d 100644
--- a/kubernetes/manager/templates/_helpers.tpl
+++ b/kubernetes/manager/templates/_helpers.tpl
@@ -109,6 +109,38 @@ Validate logging values.
{{- end -}}
{{- end }}
+{{/*
+Validate OpenTelemetry values.
+*/}}
+{{- define "manager.otel.validate" -}}
+{{- if .Values.or.otel.enabled -}}
+ {{- if not .Values.or.otel.serviceName -}}
+{{- fail "manager chart value or.otel.serviceName is required when or.otel.enabled is true" -}}
+ {{- end -}}
+ {{- if not .Values.or.otel.endpoint -}}
+{{- fail "manager chart value or.otel.endpoint is required when or.otel.enabled is true" -}}
+ {{- end -}}
+ {{- if not .Values.or.otel.protocol -}}
+{{- fail "manager chart value or.otel.protocol is required when or.otel.enabled is true" -}}
+ {{- end -}}
+ {{- $managedEnvironmentVariables := list
+ "OTEL_JAVAAGENT_ENABLED"
+ "OTEL_SERVICE_NAME"
+ "OTEL_TRACES_EXPORTER"
+ "OTEL_EXPORTER_OTLP_ENDPOINT"
+ "OTEL_EXPORTER_OTLP_PROTOCOL"
+ "OTEL_METRICS_EXPORTER"
+ "OTEL_LOGS_EXPORTER"
+ "OTEL_INSTRUMENTATION_COMMON_DB_STATEMENT_SANITIZER_ENABLED"
+ -}}
+ {{- range .Values.or.env -}}
+ {{- if has .name $managedEnvironmentVariables -}}
+{{- fail (printf "manager chart values or.otel.* manage %s; remove %s from or.env" .name .name) -}}
+ {{- end -}}
+ {{- end -}}
+{{- end -}}
+{{- end }}
+
{{/*
Return the chart-managed logging ConfigMap name.
*/}}
diff --git a/kubernetes/manager/templates/deployment.yaml b/kubernetes/manager/templates/deployment.yaml
index d67f2f2c2..d346cb1c9 100644
--- a/kubernetes/manager/templates/deployment.yaml
+++ b/kubernetes/manager/templates/deployment.yaml
@@ -1,4 +1,5 @@
{{- include "manager.logging.validate" . }}
+{{- include "manager.otel.validate" . }}
{{- $loggingInlineConfig := .Values.logging.config -}}
{{- $loggingExistingConfigMap := .Values.logging.existingConfigMap -}}
{{- $loggingEnabled := or $loggingInlineConfig $loggingExistingConfigMap -}}
@@ -137,6 +138,25 @@ spec:
- name: OR_METRICS_PORT
value: "{{ .Values.or.metricsPort }}"
+ {{- if .Values.or.otel.enabled }}
+ - name: OTEL_JAVAAGENT_ENABLED
+ value: "true"
+ - name: OTEL_SERVICE_NAME
+ value: {{ .Values.or.otel.serviceName | quote }}
+ - name: OTEL_TRACES_EXPORTER
+ value: "otlp"
+ - name: OTEL_EXPORTER_OTLP_ENDPOINT
+ value: {{ .Values.or.otel.endpoint | quote }}
+ - name: OTEL_EXPORTER_OTLP_PROTOCOL
+ value: {{ .Values.or.otel.protocol | quote }}
+ - name: OTEL_METRICS_EXPORTER
+ value: "none"
+ - name: OTEL_LOGS_EXPORTER
+ value: "none"
+ - name: OTEL_INSTRUMENTATION_COMMON_DB_STATEMENT_SANITIZER_ENABLED
+ value: "true"
+ {{- end }}
+
{{- if $loggingEnabled }}
- name: OR_LOGGING_CONFIG_FILE
value: "{{ include "manager.logging.configFilePath" . }}"
diff --git a/kubernetes/manager/values.yaml b/kubernetes/manager/values.yaml
index ac19a54ad..2588c9c65 100644
--- a/kubernetes/manager/values.yaml
+++ b/kubernetes/manager/values.yaml
@@ -171,6 +171,14 @@ or:
# - name: foo
# value: bar
+ # OpenTelemetry Java agent tracing. The endpoint must be set when tracing is enabled.
+ # Additional agent settings, such as sampling, can be supplied through or.env.
+ otel:
+ enabled: false
+ serviceName: "openremote-manager"
+ endpoint: ""
+ protocol: "http/protobuf"
+
# The public HOSTNAME and SSL PORT of this manager
hostname: "localhost"
port: "-1"
diff --git a/manager/Dockerfile b/manager/Dockerfile
index 948ab6197..89253795e 100644
--- a/manager/Dockerfile
+++ b/manager/Dockerfile
@@ -174,6 +174,7 @@ RUN mkdir -p /storage/logs
WORKDIR /opt/app
ADD lib /opt/app/lib
+ADD opentelemetry /opt/opentelemetry
ADD web /opt/web
ADD map /opt/map
diff --git a/manager/build.gradle b/manager/build.gradle
index 41f63f8e3..0a9d0ab1c 100644
--- a/manager/build.gradle
+++ b/manager/build.gradle
@@ -14,6 +14,14 @@ application {
mainClass = "org.openremote.manager.Main"
}
+configurations {
+ otelJavaAgent {
+ canBeConsumed = false
+ canBeResolved = true
+ transitive = false
+ }
+}
+
dependencies {
api project(":container")
@@ -58,6 +66,8 @@ dependencies {
implementation libs.sshdNetty
+ otelJavaAgent libs.opentelemetryJavaagent
+
testImplementation libs.groovyAll
testImplementation platform(libs.spockBom)
testImplementation libs.spockCore
@@ -137,6 +147,11 @@ distributions {
into("map") {
from("${projectDir}/src/map")
}
+ into("opentelemetry") {
+ from(configurations.otelJavaAgent) {
+ rename { "opentelemetry-javaagent.jar" }
+ }
+ }
}
}
}
diff --git a/manager/entrypoint.sh b/manager/entrypoint.sh
index 52cdd39e6..5da8242b1 100644
--- a/manager/entrypoint.sh
+++ b/manager/entrypoint.sh
@@ -10,4 +10,12 @@ if [ -n "$HEAP_DUMPS" ]; then
echo "WARNING: Heap dump(s) found in /storage - remove them to free up disk space:"
ls -lh /storage/dump_*.hprof
fi
-exec java ${JAVA_OPTS} ${JAVA_OPTS_APPEND} -cp "/opt/app/lib/*:/deployment/manager/extensions/*:/extensions/*" org.openremote.manager.Main
+
+OTEL_JAVA_AGENT_OPTION=
+case "${OTEL_JAVAAGENT_ENABLED:-false}" in
+ [Tt][Rr][Uu][Ee])
+ OTEL_JAVA_AGENT_OPTION="-javaagent:/opt/opentelemetry/opentelemetry-javaagent.jar"
+ ;;
+esac
+
+exec java ${JAVA_OPTS} ${JAVA_OPTS_APPEND} ${OTEL_JAVA_AGENT_OPTION} -cp "/opt/app/lib/*:/deployment/manager/extensions/*:/extensions/*" org.openremote.manager.Main
diff --git a/profile/demo.yml b/profile/demo.yml
index ad4ec092f..293116c5b 100644
--- a/profile/demo.yml
+++ b/profile/demo.yml
@@ -114,6 +114,17 @@ services:
OR_ADDITIONAL_HOSTNAMES:
OR_WEBSERVER_ALLOWED_ORIGINS: ${OR_WEBSERVER_ALLOWED_ORIGINS:-}
OR_DEV_MODE: ${OR_DEV_MODE:-false}
+
+ # Opt-in OpenTelemetry tracing to a local Alloy OTLP/HTTP receiver.
+ OTEL_JAVAAGENT_ENABLED: ${OTEL_JAVAAGENT_ENABLED:-false}
+ OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-openremote-manager}
+ OTEL_TRACES_EXPORTER: ${OTEL_TRACES_EXPORTER:-otlp}
+ OTEL_EXPORTER_OTLP_ENDPOINT:
+ OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
+ OTEL_METRICS_EXPORTER: ${OTEL_METRICS_EXPORTER:-none}
+ OTEL_LOGS_EXPORTER: ${OTEL_LOGS_EXPORTER:-none}
+ OTEL_INSTRUMENTATION_COMMON_DB_STATEMENT_SANITIZER_ENABLED: "true"
+
# The following variables will configure the demo
OR_FORECAST_SOLAR_API_KEY:
OR_OPEN_WEATHER_API_APP_ID:
diff --git a/profile/deploy.yml b/profile/deploy.yml
index 80c700f76..464c96d19 100644
--- a/profile/deploy.yml
+++ b/profile/deploy.yml
@@ -337,6 +337,19 @@ services:
# Set the number of concurrent attribute event processors (defaults to number of CPU cores)
OR_ATTRIBUTE_EVENT_THREADS:
+ # Opt-in OpenTelemetry tracing. The Java agent is packaged in the manager image but is only attached when
+ # explicitly enabled. Set OTEL_EXPORTER_OTLP_ENDPOINT outside this file; for an Alloy service in this Compose
+ # project it will usually be http://alloy-otel:4318.
+ OTEL_JAVAAGENT_ENABLED: ${OTEL_JAVAAGENT_ENABLED:-false}
+ OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-openremote-manager}
+ OTEL_TRACES_EXPORTER: ${OTEL_TRACES_EXPORTER:-otlp}
+ OTEL_EXPORTER_OTLP_ENDPOINT:
+ OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
+ # This integration is traces-only; existing Prometheus metrics and application logging remain unchanged.
+ OTEL_METRICS_EXPORTER: ${OTEL_METRICS_EXPORTER:-none}
+ OTEL_LOGS_EXPORTER: ${OTEL_LOGS_EXPORTER:-none}
+ OTEL_INSTRUMENTATION_COMMON_DB_STATEMENT_SANITIZER_ENABLED: "true"
+
# Set the persistence storage dir (must also be volume mapped)
OR_STORAGE_DIR: