100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Metrics instrumentation, global lock removal and camel route cleanup (#1080)
Rich Turnerf55d41103aa3
diff --git a/.ci_cd/aws/cloudformation-create-ec2.yml b/.ci_cd/aws/cloudformation-create-ec2.ymlindex ca0461f2b..e82c6c621 100644--- a/.ci_cd/aws/cloudformation-create-ec2.yml+++ b/.ci_cd/aws/cloudformation-create-ec2.yml@@ -201,8 +201,10 @@ Resources:AWS::CloudFormation::Init:configSets:default:- - 01_setupCfnHup- - 02_restart_amazon-cloudwatch-agent+ - setupCfnHup+ - create_swapfile+ - cloudwatch_config+ - restart_amazon-cloudwatch-agent- docker_setup- cron_setup- env_vars@@ -211,16 +213,213 @@ Resources:- deployment_local_dir- daily_cleanup+ create_swapfile:+ commands:+ 01_swapfile:+ command: |+ if [ -f "/swapfile" ]; then+ echo "Swapfile already exists"+ exit 0+ fi++ dd if=/dev/zero of=/swapfile bs=128M count=16+ chmod 600 /swapfile+ mkswap /swapfile+ swapon /swapfile+ echo "/swapfile swap swap defaults 0 0" >> /etc/fstab+++ cloudwatch_config:+ files:+ "/opt/aws/amazon-cloudwatch-agent/var/config.json":+ content: |+ {+ "metrics": {+ "append_dimensions": {+ "AutoScalingGroupName": "${aws:AutoScalingGroupName}",+ "ImageId": "${aws:ImageId}",+ "InstanceId": "${aws:InstanceId}",+ "InstanceType": "${aws:InstanceType}"+ },+ "metrics_collected": {+ "mem": {+ "measurement": [+ "mem_used_percent"+ ],+ "metrics_collection_interval": 300+ },+ "disk": {+ "drop_device": true,+ "measurement": [+ "used_percent"+ ],+ "resources": [+ "/"+ ],+ "metrics_collection_interval": 600+ }+ }+ },+ "logs": {+ "metrics_collected": {+ "prometheus": {+ "log_group_name": "Prometheus",+ "prometheus_config_path": "/opt/aws/amazon-cloudwatch-agent/var/prometheus.yaml",+ "emf_processor": {+ "metric_declaration_dedup": true,+ "metric_namespace": "CWAgent-Prometheus",+ "metric_unit": {+ "or_rules_facts": "Count",+ "or_rules_firing_sum": "Seconds",+ "or_rules_firing_count": "Count",+ "or_camel_route_queue_size": "Count",+ "or_camel_route_failed_total": "Count",+ "or_camel_route_total": "Count",+ "or_camel_route_seconds_sum": "Seconds",+ "or_camel_route_seconds_count": "Count",+ "executor_seconds_count": "Count",+ "executor_seconds_sum": "Seconds",+ "executor_queued_tasks": "Count",+ "haproxy_server_current_sessions": "Count",+ "haproxy_server_bytes_in_total": "Bytes",+ "haproxy_server_bytes_out_total": "Bytes",+ "haproxy_server_status": "Count",+ "haproxy_server_http_responses_total": "Count",+ "haproxy_server_max_session_rate": "Count/Second",+ "haproxy_server_total_time_average_seconds": "Seconds"+ },+ "metric_declaration": [+ {+ "source_labels": [ "job" ],+ "label_matcher": "^manager$",+ "dimensions": [+ [ "instance_name" ]+ ],+ "metric_selectors": [+ "^or_rules.*$"+ ]+ },+ {+ "source_labels": [ "job" ],+ "label_matcher": "^manager$",+ "dimensions": [+ [ "instance_name","routeId" ]+ ],+ "metric_selectors": [+ "^or_camel_route.*$"+ ]+ },+ {+ "source_labels": [ "job" ],+ "label_matcher": "^manager$",+ "dimensions": [+ [ "instance_name","name" ]+ ],+ "metric_selectors": [+ "^executor_seconds.*$",+ "^executor_queued.*$"+ ]+ },+ {+ "source_labels": [ "job" ],+ "label_matcher": "^proxy$",+ "dimensions": [+ [ "instance_name", "proxy", "server" ]+ ],+ "metric_selectors": [+ "^haproxy_server_total_time_average_seconds$",+ "^haproxy_server_max_session_rate$",+ "^haproxy_server_bytes.*$",+ "^haproxy_server_current_sessions$"+ ]+ },+ {+ "source_labels": [ "job" ],+ "label_matcher": "^proxy$",+ "dimensions": [+ [ "instance_name", "proxy", "server", "code" ]+ ],+ "metric_selectors": [+ "^haproxy_server_http_responses_total$"+ ]+ },+ {+ "source_labels": [ "job" ],+ "label_matcher": "^proxy$",+ "dimensions": [+ [ "instance_name", "proxy", "server", "state" ]+ ],+ "metric_selectors": [+ "^haproxy_server_status$"+ ]+ }+ ]+ }+ }+ }+ }+ }++ "/opt/aws/amazon-cloudwatch-agent/var/prometheus.yaml":+ content: !Sub |+ # Prometheus scrape config file+ global:+ scrape_interval: 1m+ scrape_timeout: 10s+ scrape_configs:+ - job_name: proxy+ sample_limit: 10000+ static_configs:+ - targets:+ - localhost:8404+ - labels:+ instance_name: '${Host}' # DOESN'T WORK https://github.com/aws/amazon-cloudwatch-agent/issues/751+ metric_relabel_configs:+ - action: keep+ source_labels: [__name__,state,code]+ regex: '(haproxy_server_current_sessions;;|haproxy_server_bytes_in_total;;|haproxy_server_bytes_out_total;;|haproxy_server_status;UP;|haproxy_server_http_responses_total;;5xx|haproxy_server_max_session_rate;;|haproxy_server_total_time_average_seconds;;)'+ - action: replace # THIS IS A WORKAROUND FOR BROKEN LABELS ABOVE+ source_labels: [ instance_name ]+ regex: '(.*)'+ replacement: '${Host}'+ target_label: instance_name++ - job_name: manager+ sample_limit: 10000+ static_configs:+ - targets:+ - localhost:8405+ - labels:+ instance_name: '${Host}' # DOESN'T WORK https://github.com/aws/amazon-cloudwatch-agent/issues/751+ metric_relabel_configs:+ - action: keep+ source_labels: [__name__]+ regex: '(executor_queued_tasks|or_camel_route_queue_size|or_camel_route_failed_total|or_camel_route_total|or_camel_route_seconds.*|or_rules_.*)'+ - action: replace+ source_labels: [name]+ regex: '.+:\/\/(.+)\?.*'+ replacement: '${!1}'+ target_label: name+ - action: replace # THIS IS A WORKAROUND FOR BROKEN LABELS ABOVE+ source_labels: [instance_name]+ regex: '(.*)'+ replacement: '${Host}'+ target_label: instance_name+daily_cleanup:files:"/usr/local/bin/daily-cleanup":content: !Sub |#!/bin/bash- journalctl --vacuum-size=500M+ journalctl --vacuum-time=7d"/etc/cron.d/cleanup":content: |#!/bin/bash0 3 * * * root /usr/local/bin/daily-cleanup+ commands:+ 01_chmod:+ command: |+ chmod +x /usr/local/bin/daily-cleanupdeployment_local_dir:commands:@@ -531,14 +730,15 @@ Resources:enabled: "true"ensureRunning: "true"- 02_restart_amazon-cloudwatch-agent:+ restart_amazon-cloudwatch-agent:commands:01_stop_service:command: /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop02_start_service:command: |- /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c ssm:CloudWatch-Config -s- 01_setupCfnHup:+ /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/var/config.json++ setupCfnHup:files:'/etc/cfn/cfn-hup.conf':content: !Sub |@@ -603,13 +803,11 @@ Resources:#sed -i '/download_updates/s/= .*/= no/' /etc/yum/yum-cron.confrpm -Uvh https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/arm64/latest/amazon-cloudwatch-agent.rpm- /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c ssm:CloudWatch-Config -s+ /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/var/config.json# Execute required config set/opt/aws/bin/cfn-init -v --stack ${AWS::StackName} --resource EC2Instance --configsets default --region ${AWS::Region}- # Swap file necessary?-# All done so signal success/opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackId} --resource EC2Instance --region ${AWS::Region}@@ -670,7 +868,7 @@ Resources:MetricName: StatusCheckFailed_InstanceStatistic: MinimumPeriod: 60- EvaluationPeriods: 3+ EvaluationPeriods: 5ComparisonOperator: GreaterThanThresholdThreshold: 0AlarmActions:@@ -693,7 +891,6 @@ Resources:ComparisonOperator: GreaterThanThresholdThreshold: 90AlarmActions:- - !Sub "arn:aws:automate:${AWS::Region}:ec2:reboot"- !Ref SnsTopicOKActions:- !Ref SnsTopic@@ -748,3 +945,236 @@ Resources:Value: !Ref ImageId- Name: InstanceTypeValue: !Ref InstanceType++ CloudWatchDashboard:+ Type: AWS::CloudWatch::Dashboard+ Properties:+ DashboardName: !Join ["-", !Split [ '.', !Ref Host] ]+ DashboardBody: !Sub |+ {+ "widgets": [+ {+ "height": 6,+ "width": 6,+ "y": 0,+ "x": 0,+ "type": "metric",+ "properties": {+ "metrics": [+ [ "CWAgent-Prometheus", "executor_queued_tasks", "instance_name", "${Host}", "name", "ContainerExecutorService", { "label": "ContainerExecutorService" } ]+ ],+ "view": "timeSeries",+ "stacked": false,+ "region": "${AWS::Region}",+ "period": 300,+ "title": "Thread Pool Queue Size",+ "stat": "Average"+ }+ },+ {+ "height": 6,+ "width": 6,+ "y": 0,+ "x": 6,+ "type": "metric",+ "properties": {+ "metrics": [+ [ { "expression": "m1 / m2", "label": "Avg Processing Time", "id": "e1" } ],+ [ "CWAgent-Prometheus", "or_camel_route_queue_size", "routeId", "AttributeEventProcessor", "instance_name", "${Host}", { "id": "e2", "label": "Queued Events", "yAxis": "right" } ],+ [ ".", "or_camel_route_seconds_sum", ".", ".", ".", ".", { "id": "m1", "visible": false, "stat": "Average" } ],+ [ ".", "or_camel_route_seconds_count", ".", ".", ".", ".", { "id": "m2", "visible": false, "stat": "Average" } ]+ ],+ "view": "timeSeries",+ "stacked": false,+ "region": "${AWS::Region}",+ "stat": "Maximum",+ "period": 300,+ "title": "Attr Event Processing Time"+ }+ },+ {+ "height": 6,+ "width": 6,+ "y": 6,+ "x": 6,+ "type": "metric",+ "properties": {+ "metrics": [+ [ { "expression": "RUNNING_SUM(m1)", "label": "Total", "id": "e1" } ],+ [ { "expression": "RUNNING_SUM(m2)", "label": "Failed", "id": "e2" } ],+ [ "CWAgent-Prometheus", "or_camel_route_total", "routeId", "AttributeEventProcessor", "instance_name", "${Host}", { "id": "m1", "visible": false } ],+ [ "CWAgent-Prometheus", "or_camel_route_failed_total", "routeId", "AttributeEventProcessor", "instance_name", "${Host}", { "id": "m2", "visible": false } ]+ ],+ "view": "timeSeries",+ "stacked": false,+ "region": "${AWS::Region}",+ "title": "Attr Event Count",+ "stat": "Maximum",+ "period": 300+ }+ },+ {+ "height": 6,+ "width": 6,+ "y": 6,+ "x": 0,+ "type": "metric",+ "properties": {+ "metrics": [+ [ { "expression": "m2 / m1", "label": "All", "id": "e1" } ],+ [ "CWAgent-Prometheus", "or_rules_firing_count", "instance_name", "${Host}", { "id": "m1", "visible": false } ],+ [ "CWAgent-Prometheus", "or_rules_firing_sum", "instance_name", "${Host}", { "id": "m2", "visible": false } ]+ ],+ "view": "timeSeries",+ "stacked": false,+ "region": "${AWS::Region}",+ "stat": "Average",+ "period": 300,+ "title": "Rule Engine Firing Duration (Seconds)"+ }+ },+ {+ "height": 6,+ "width": 6,+ "y": 6,+ "x": 12,+ "type": "metric",+ "properties": {+ "metrics": [+ [ "CWAgent-Prometheus", "haproxy_server_current_sessions", "server", "manager", "proxy", "mqtt", "instance_name", "${Host}", { "label": "MQTT" } ],+ [ "...", "manager_backend", ".", ".", { "label": "HTTP/WS" } ],+ [ "...", "keycloak", ".", "keycloak_backend", ".", ".", { "label": "Keycloak" } ]+ ],+ "view": "timeSeries",+ "stacked": false,+ "region": "${AWS::Region}",+ "title": "Proxy Active Sessions",+ "stat": "Maximum",+ "period": 300+ }+ },+ {+ "height": 6,+ "width": 6,+ "y": 0,+ "x": 12,+ "type": "metric",+ "properties": {+ "metrics": [+ [ "CWAgent-Prometheus", "haproxy_server_max_session_rate", "server", "manager", "proxy", "mqtt", "instance_name", "${Host}", { "label": "MQTT" } ],+ [ "...", "manager_backend", ".", ".", { "label": "HTTP/WS" } ],+ [ "...", "keycloak", ".", "keycloak_backend", ".", ".", { "label": "Keycloak" } ]+ ],+ "view": "timeSeries",+ "stacked": false,+ "region": "${AWS::Region}",+ "title": "Proxy Session Rate",+ "stat": "Average",+ "period": 300+ }+ },+ {+ "height": 6,+ "width": 6,+ "y": 12,+ "x": 12,+ "type": "metric",+ "properties": {+ "metrics": [+ [ "CWAgent-Prometheus", "haproxy_server_bytes_in_total", "server", "manager", "proxy", "mqtt", "instance_name", "${Host}", { "label": "MQTT Bytes In" } ],+ [ ".", "haproxy_server_bytes_out_total", ".", ".", ".", ".", ".", ".", { "label": "MQTT Bytes Out" } ],+ [ "...", "keycloak", ".", "keycloak_backend", ".", ".", { "label": "Keycloak Bytes Out" } ],+ [ ".", "haproxy_server_bytes_in_total", ".", ".", ".", ".", ".", ".", { "label": "Keycloak Bytes In" } ],+ [ "...", "manager", ".", "manager_backend", ".", ".", { "label": "HTTP/WS Bytes In" } ],+ [ ".", "haproxy_server_bytes_out_total", ".", ".", ".", ".", ".", ".", { "label": "HTTP/WS Bytes Out" } ]+ ],+ "view": "timeSeries",+ "stacked": false,+ "region": "${AWS::Region}",+ "title": "Proxy I/O",+ "stat": "Average",+ "period": 300+ }+ },+ {+ "height": 6,+ "width": 6,+ "y": 12,+ "x": 6,+ "type": "metric",+ "properties": {+ "metrics": [+ [ "CWAgent-Prometheus", "haproxy_server_http_responses_total", "server", "keycloak", "proxy", "keycloak_backend", "instance_name", "${Host}", "code", "5xx", { "label": "Keycloak 5xx" } ],+ [ "...", "manager", ".", "manager_backend", ".", ".", ".", ".", { "label": "Manager 5xx" } ]+ ],+ "view": "timeSeries",+ "stacked": false,+ "region": "${AWS::Region}",+ "title": "HTTP Error Responses",+ "stat": "Maximum",+ "period": 300+ }+ },+ {+ "height": 6,+ "width": 6,+ "y": 6,+ "x": 18,+ "type": "metric",+ "properties": {+ "metrics": [+ [ "AWS/EBS", "VolumeReadOps", "VolumeId", "vol-08af38ccf9207da16", { "label": "Reads" } ],+ [ ".", "VolumeWriteOps", ".", ".", { "label": "Writes" } ]+ ],+ "view": "timeSeries",+ "stacked": false,+ "region": "${AWS::Region}",+ "title": "EBS Operations",+ "stat": "Average",+ "period": 300+ }+ },+ {+ "height": 6,+ "width": 6,+ "y": 12,+ "x": 18,+ "type": "metric",+ "properties": {+ "metrics": [+ [ "AWS/EBS", "VolumeReadBytes", "VolumeId", "vol-08af38ccf9207da16", { "label": "Read" } ],+ [ ".", "VolumeWriteBytes", ".", ".", { "label": "Write" } ]+ ],+ "view": "timeSeries",+ "stacked": false,+ "region": "${AWS::Region}",+ "title": "EBS Bytes",+ "stat": "Average",+ "period": 300+ }+ },+ {+ "height": 6,+ "width": 6,+ "y": 0,+ "x": 18,+ "type": "metric",+ "properties": {+ "metrics": [+ [ "AWS/EC2", "EBSIOBalance%", "InstanceId", "${EC2Instance}", { "label": "EBS IO Balance %" } ],+ [ ".", "EBSByteBalance%", ".", ".", { "label": "EBS Byte Balance %" } ],+ [ ".", "CPUCreditBalance", ".", ".", { "yAxis": "right", "label": "CPU Credit Balance" } ],+ [ ".", "CPUUtilization", ".", ".", { "label": "CPU Utilization %", "stat": "Maximum" } ],+ [ "CWAgent", "mem_used_percent", ".", ".", "ImageId", "${ImageId}", "InstanceType", "${InstanceType}", { "label": "Memory Usage %", "stat": "Average" } ],+ [ ".", "disk_used_percent", "path", "/", "InstanceId", "i-0c9e821c9865e70ea", "ImageId", "${ImageId}", "InstanceType", "${InstanceType}", "fstype", "xfs", { "label": "Disk Usage %", "stat": "Average" } ]+ ],+ "view": "timeSeries",+ "stacked": false,+ "region": "${AWS::Region}",+ "stat": "Minimum",+ "period": 300,+ "title": "EC2 Stats"+ }+ }+ ]+ }diff --git a/.ci_cd/host_init/init.sh b/.ci_cd/host_init/init.shindex c8e78ec4d..b60dfec21 100644--- a/.ci_cd/host_init/init.sh+++ b/.ci_cd/host_init/init.sh@@ -33,7 +33,7 @@ if [ "$DAILY_RESTART" == 'true' ]; thenecho "Adding daily manager restart cron job"echo '#!/bin/bash' > /etc/cron.d/or-restart- echo "0 5 * * * root $HOME_DIR/temp/host_init/restart.sh" >> /etc/cron.d/or-restart+ echo "0 3 * * * root $HOME_DIR/temp/host_init/restart.sh" >> /etc/cron.d/or-restartelseecho "Removing any existing daily restart cron job"rm -f /etc/cron.d/or-restart &>/dev/nulldiff --git a/agent/src/main/java/org/openremote/agent/protocol/AbstractProtocol.java b/agent/src/main/java/org/openremote/agent/protocol/AbstractProtocol.javaindex f4932c621..bb882fac4 100644--- a/agent/src/main/java/org/openremote/agent/protocol/AbstractProtocol.java+++ b/agent/src/main/java/org/openremote/agent/protocol/AbstractProtocol.java@@ -22,7 +22,6 @@ package org.openremote.agent.protocol;import org.apache.camel.ProducerTemplate;import org.apache.camel.builder.RouteBuilder;import org.apache.camel.impl.DefaultCamelContext;-import org.openremote.container.concurrent.GlobalLock;import org.openremote.container.message.MessageBrokerService;import org.openremote.container.timer.TimerService;import org.openremote.model.Container;@@ -38,46 +37,23 @@ import org.openremote.model.attribute.AttributeRef;import org.openremote.model.attribute.AttributeState;import org.openremote.model.protocol.ProtocolUtil;import org.openremote.model.query.AssetQuery;-import org.openremote.model.syslog.SyslogCategory;import org.openremote.model.util.Pair;import org.openremote.model.util.TextUtil;import java.util.*;+import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ScheduledExecutorService;import java.util.concurrent.TimeUnit;import java.util.function.Consumer;-import java.util.logging.Logger;-import static org.openremote.container.concurrent.GlobalLock.withLock;import static org.openremote.model.protocol.ProtocolUtil.hasDynamicWriteValue;import static org.openremote.model.syslog.SyslogCategory.PROTOCOL;-/**- * Thread-safe base implementation for protocols.- * <p>- * Subclasses should use the {@link GlobalLock#withLock} and {@link GlobalLock#withLockReturning} methods- * to guard critical sections when modifying shared state:- * <blockquote><pre>{@code- * withLock(getProtocolName(), () -> {- * // Critical section- * });- * }</pre></blockquote>- * <blockquote><pre>{@code- * return withLockReturning(() -> {- * // Critical section- * return ...;- * });- * }</pre></blockquote>- * <p>- * All <code>abstract</code> methods are always called within lock scope. An implementation can rely on this lock- * and safely modify internal, protocol-specific shared state. However, if a protocol implementation schedules- * an asynchronous task, this task must obtain the lock to call any protocol operations.- */public abstract class AbstractProtocol<T extends Agent<T, ?, U>, U extends AgentLink<?>> implements Protocol<T> {- private static final Logger LOG = SyslogCategory.getLogger(PROTOCOL, AbstractProtocol.class);- protected final Map<AttributeRef, Attribute<?>> linkedAttributes = new HashMap<>();- protected final Set<AttributeRef> dynamicAttributes = new HashSet<>();+ private static final System.Logger LOG = System.getLogger(AbstractProtocol.class.getSimpleName() + "." + PROTOCOL.name());+ protected final Map<AttributeRef, Attribute<?>> linkedAttributes = new ConcurrentHashMap<>();+ protected final Set<AttributeRef> dynamicAttributes = Collections.synchronizedSet(new HashSet<>());protected DefaultCamelContext messageBrokerContext;protected ProducerTemplate producerTemplate;protected TimerService timerService;@@ -100,55 +76,53 @@ public abstract class AbstractProtocol<T extends Agent<T, ?, U>, U extends AgentdatapointService = container.getService(ProtocolDatapointService.class);messageBrokerContext = container.getService(MessageBrokerService.class).getContext();- withLock(getProtocolName() + "::start", () -> {- try {- messageBrokerContext.addRoutes(new RouteBuilder() {- @Override- public void configure() throws Exception {- from(ACTUATOR_TOPIC)- .routeId("Actuator-" + getProtocolName() + getAgent().getId())- .process(exchange -> {- Protocol<?> protocolInstance = exchange.getIn().getHeader(ACTUATOR_TOPIC_TARGET_PROTOCOL, Protocol.class);- if (protocolInstance != AbstractProtocol.this) {- return;- }-+ // TODO: Priority should each protocol instance have its' own camel route+ try {+ messageBrokerContext.addRoutes(new RouteBuilder() {+ @Override+ public void configure() throws Exception {+ from(ACTUATOR_TOPIC)+ .routeId("ProtocolInbound-" + getProtocolName() + getAgent().getId())+ .process(exchange -> {+ final Protocol<?> protocolInstance = exchange.getIn().getHeader(ACTUATOR_TOPIC_TARGET_PROTOCOL, Protocol.class);+ if (protocolInstance != AbstractProtocol.this) {+ return;+ }+ synchronized (this) {AttributeEvent event = exchange.getIn().getBody(AttributeEvent.class);Attribute<?> linkedAttribute = getLinkedAttributes().get(event.getAttributeRef());if (linkedAttribute == null) {- LOG.info("Attempt to write to attribute that is not actually linked to this protocol '" + AbstractProtocol.this + "': " + linkedAttribute);+ LOG.log(System.Logger.Level.INFO, () -> "Attempt to write to attribute that is not actually linked to this protocol '" + AbstractProtocol.this + "': " + event);return;}- processLinkedAttributeWrite(event);- });- }- });+ processLinkedAttributeWrite(linkedAttribute, event);+ }+ });+ }+ });- doStart(container);+ doStart(container);- } catch (Exception ex) {- throw new RuntimeException(ex);- }- });+ } catch (Exception ex) {+ throw new RuntimeException(ex);+ }this.producerTemplate = container.getService(MessageBrokerService.class).getProducerTemplate();}@Overridefinal public void stop(Container container) {- withLock(getProtocolName() + "::stop", () -> {- linkedAttributes.clear();- try {- messageBrokerContext.stopRoute("Actuator-" + getProtocolName(), 1, TimeUnit.MILLISECONDS);- messageBrokerContext.removeRoute("Actuator-" + getProtocolName());+ linkedAttributes.clear();+ try {+ messageBrokerContext.stopRoute("Actuator-" + getProtocolName(), 1, TimeUnit.MILLISECONDS);+ messageBrokerContext.removeRoute("Actuator-" + getProtocolName());- doStop(container);+ doStop(container);- } catch (Exception ex) {- throw new RuntimeException(ex);- }- });+ } catch (Exception ex) {+ throw new RuntimeException(ex);+ }}protected void setConnectionStatus(ConnectionStatus connectionStatus) {@@ -157,44 +131,34 @@ public abstract class AbstractProtocol<T extends Agent<T, ?, U>, U extends Agent@Overridefinal public void linkAttribute(String assetId, Attribute<?> attribute) throws Exception {- withLock(getProtocolName() + "::linkAttribute", () -> {-- AttributeRef attributeRef = new AttributeRef(assetId, attribute.getName());-- if (linkedAttributes.containsKey(attributeRef)) {- LOG.warning("Attribute is already linked to this protocol so ignoring: " + attributeRef);- return;- }+ AttributeRef attributeRef = new AttributeRef(assetId, attribute.getName());- // Need to add to map before actual linking as protocols may want to update the value as part of- // linking process and without entry in the map any update would be blocked- linkedAttributes.put(attributeRef, attribute);+ // Need to add to map before actual linking as protocols may want to update the value as part of+ // linking process and without entry in the map any update would be blocked+ linkedAttributes.put(attributeRef, attribute);- // Check for dynamic value placeholder+ // Check for dynamic value placeholder- if (hasDynamicWriteValue(agent.getAgentLink(attribute))) {- dynamicAttributes.add(attributeRef);- }+ if (hasDynamicWriteValue(agent.getAgentLink(attribute))) {+ dynamicAttributes.add(attributeRef);+ }- try {- doLinkAttribute(assetId, attribute, agent.getAgentLink(attribute));- } catch (Exception e) {- linkedAttributes.remove(attributeRef);- throw new RuntimeException(e);- }- });+ try {+ doLinkAttribute(assetId, attribute, agent.getAgentLink(attribute));+ } catch (Exception e) {+ linkedAttributes.remove(attributeRef);+ throw new RuntimeException(e);+ }}@Overridefinal public void unlinkAttribute(String assetId, Attribute<?> attribute) throws Exception {- withLock(getProtocolName() + "::unlinkAttributes", () -> {- AttributeRef attributeRef = new AttributeRef(assetId, attribute.getName());+ AttributeRef attributeRef = new AttributeRef(assetId, attribute.getName());- if (linkedAttributes.remove(attributeRef) != null) {- dynamicAttributes.remove(attributeRef);- doUnlinkAttribute(assetId, attribute, agent.getAgentLink(attribute));- }- });+ if (linkedAttributes.remove(attributeRef) != null) {+ dynamicAttributes.remove(attributeRef);+ doUnlinkAttribute(assetId, attribute, agent.getAgentLink(attribute));+ }}public T getAgent() {@@ -206,37 +170,27 @@ public abstract class AbstractProtocol<T extends Agent<T, ?, U>, U extends Agentreturn linkedAttributes;}- final protected void processLinkedAttributeWrite(AttributeEvent event) {- LOG.finest("Processing linked attribute write on protocol '" + this + "': " + event);- withLock(getProtocolName() + "::processLinkedAttributeWrite", () -> {-- Attribute<?> attribute = linkedAttributes.get(event.getAttributeRef());+ final protected void processLinkedAttributeWrite(Attribute<?> attribute, AttributeEvent event) {+ LOG.log(System.Logger.Level.TRACE, () -> "Processing linked attribute write on protocol '" + this + "': " + event);+ AgentLink<?> agentLink = agent.getAgentLink(attribute);- if (attribute == null) {- LOG.warning("Attribute not linked to protocol '" + this + "':" + event);- } else {+ Pair<Boolean, Object> ignoreAndConverted = ProtocolUtil.doOutboundValueProcessing(+ event.getAssetId(),+ attribute,+ agentLink,+ event.getValue().orElse(null),+ dynamicAttributes.contains(event.getAttributeRef()));- AgentLink<?> agentLink = agent.getAgentLink(attribute);-- Pair<Boolean, Object> ignoreAndConverted = ProtocolUtil.doOutboundValueProcessing(- event.getAssetId(),- attribute,- agentLink,- event.getValue().orElse(null),- dynamicAttributes.contains(event.getAttributeRef()));-- if (ignoreAndConverted.key) {- LOG.fine("Value conversion returned ignore so attribute will not write to protocol: " + event.getAttributeRef());- return;- }+ if (ignoreAndConverted.key) {+ LOG.log(System.Logger.Level.DEBUG, "Value conversion returned ignore so attribute will not write to protocol: " + event.getAttributeRef());+ return;+ }- doLinkedAttributeWrite(attribute, agent.getAgentLink(attribute), event, ignoreAndConverted.value);+ doLinkedAttributeWrite(attribute, agent.getAgentLink(attribute), event, ignoreAndConverted.value);- if (agent.isUpdateOnWrite().orElse(false) || agentLink.getUpdateOnWrite().orElse(false)) {- updateLinkedAttribute(new AttributeState(event.getAttributeRef(), ignoreAndConverted.value));- }- }- });+ if (agent.isUpdateOnWrite().orElse(false) || agentLink.getUpdateOnWrite().orElse(false)) {+ updateLinkedAttribute(new AttributeState(event.getAttributeRef(), ignoreAndConverted.value));+ }}/**@@ -253,14 +207,12 @@ public abstract class AbstractProtocol<T extends Agent<T, ?, U>, U extends Agent* publish new sensor values, which performs additional verification and uses a different messaging queue.*/final protected void sendAttributeEvent(AttributeEvent event) {- withLock(getProtocolName() + "::sendAttributeEvent", () -> {- // Don't allow updating linked attributes with this mechanism as it could cause an infinite loop- if (linkedAttributes.containsKey(event.getAttributeRef())) {- LOG.warning("Cannot update an attribute linked to the same protocol; use updateLinkedAttribute for that: " + event);- return;- }- assetService.sendAttributeEvent(event);- });+ // Don't allow updating linked attributes with this mechanism as it could cause an infinite loop+ if (linkedAttributes.containsKey(event.getAttributeRef())) {+ LOG.log(System.Logger.Level.WARNING, () -> "Cannot update an attribute linked to the same protocol; use updateLinkedAttribute for that: " + event);+ return;+ }+ assetService.sendAttributeEvent(event);}/**@@ -271,19 +223,19 @@ public abstract class AbstractProtocol<T extends Agent<T, ?, U>, U extends AgentAttribute<?> attribute = linkedAttributes.get(state.getRef());if (attribute == null) {- LOG.severe("Update linked attribute called for un-linked attribute: " + state);+ LOG.log(System.Logger.Level.WARNING, () -> "Update linked attribute called for un-linked attribute: " + state);return;}Pair<Boolean, Object> ignoreAndConverted = ProtocolUtil.doInboundValueProcessing(state.getRef().getId(), attribute, agent.getAgentLink(attribute), state.getValue().orElse(null));if (ignoreAndConverted.key) {- LOG.fine("Value conversion returned ignore so attribute will not be updated: " + state.getRef());+ LOG.log(System.Logger.Level.DEBUG, "Value conversion returned ignore so attribute will not be updated: " + state.getRef());return;}AttributeEvent attributeEvent = new AttributeEvent(new AttributeState(state.getRef(), ignoreAndConverted.value), timestamp);- LOG.finest("Sending linked attribute update on sensor queue: " + attributeEvent);+ LOG.log(System.Logger.Level.TRACE, () -> "Sending linked attribute update: " + attributeEvent);producerTemplate.sendBodyAndHeader(SENSOR_QUEUE, attributeEvent, Protocol.SENSOR_QUEUE_SOURCE_PROTOCOL, getProtocolName());}@@ -292,11 +244,11 @@ public abstract class AbstractProtocol<T extends Agent<T, ?, U>, U extends Agent*/final protected void updateAgentAttribute(final AttributeState state) {if (!agent.getAttributes().has(state.getRef().getName()) || !agent.getId().equals(state.getRef().getId())) {- LOG.warning("Attempt to update non existent agent attribute or agent ID is incorrect: " + state);+ LOG.log(System.Logger.Level.WARNING, () -> "Attempt to update non existent agent attribute or agent ID is incorrect: " + state);return;}AttributeEvent attributeEvent = new AttributeEvent(state, timerService.getCurrentTimeMillis());- LOG.finest("Sending protocol agent attribute update: " + attributeEvent);+ LOG.log(System.Logger.Level.TRACE, () -> "Sending protocol agent attribute update: " + attributeEvent);assetService.sendAttributeEvent(attributeEvent);}diff --git a/agent/src/main/java/org/openremote/agent/protocol/http/HTTPProtocol.java b/agent/src/main/java/org/openremote/agent/protocol/http/HTTPProtocol.javaindex 9ab815f0f..a89261396 100644--- a/agent/src/main/java/org/openremote/agent/protocol/http/HTTPProtocol.java+++ b/agent/src/main/java/org/openremote/agent/protocol/http/HTTPProtocol.java@@ -19,6 +19,14 @@*/package org.openremote.agent.protocol.http;+import jakarta.ws.rs.HttpMethod;+import jakarta.ws.rs.client.Entity;+import jakarta.ws.rs.client.Invocation;+import jakarta.ws.rs.client.WebTarget;+import jakarta.ws.rs.core.MediaType;+import jakarta.ws.rs.core.MultivaluedHashMap;+import jakarta.ws.rs.core.MultivaluedMap;+import jakarta.ws.rs.core.Response;import org.apache.http.client.utils.URIBuilder;import org.jboss.resteasy.client.jaxrs.ResteasyClient;import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget;@@ -44,14 +52,6 @@ import org.openremote.model.util.TextUtil;import org.openremote.model.util.ValueUtil;import org.openremote.model.value.ValueType;-import jakarta.ws.rs.HttpMethod;-import jakarta.ws.rs.client.Entity;-import jakarta.ws.rs.client.Invocation;-import jakarta.ws.rs.client.WebTarget;-import jakarta.ws.rs.core.MediaType;-import jakarta.ws.rs.core.MultivaluedHashMap;-import jakarta.ws.rs.core.MultivaluedMap;-import jakarta.ws.rs.core.Response;import java.lang.annotation.Annotation;import java.lang.reflect.Type;import java.net.URI;@@ -64,14 +64,13 @@ import java.util.function.Consumer;import java.util.logging.Level;import java.util.logging.Logger;-import static org.openremote.container.concurrent.GlobalLock.withLock;import static org.openremote.container.web.QueryParameterInjectorFilter.QUERY_PARAMETERS_PROPERTY;import static org.openremote.container.web.WebTargetBuilder.createClient;import static org.openremote.model.syslog.SyslogCategory.PROTOCOL;/*** This is a HTTP client protocol for communicating with HTTP servers; it uses the {@link WebTargetBuilder} factory to- * generate JAX-RS {@link javax.ws.rs.client.WebTarget}s that can be used to make arbitrary calls to endpoints on a HTTP+ * generate JAX-RS {@link jakarta.ws.rs.client.WebTarget}s that can be used to make arbitrary calls to endpoints on a HTTP* server but it can also be extended and used as a JAX-RS client proxy.* <h1>Response filtering</h1>* <p>@@ -599,12 +598,10 @@ public class HTTPProtocol extends AbstractProtocol<HTTPAgent, HTTPAgentLink> {}protected void cancelPolling(AttributeRef attributeRef) {- withLock(getProtocolName() + "::cancelPolling", () -> {- ScheduledFuture<?> pollTask = pollingMap.remove(attributeRef);- if (pollTask != null) {- pollTask.cancel(false);- }- });+ ScheduledFuture<?> pollTask = pollingMap.remove(attributeRef);+ if (pollTask != null) {+ pollTask.cancel(false);+ }}}diff --git a/agent/src/main/java/org/openremote/agent/protocol/mqtt/AbstractMQTT_IOClient.java b/agent/src/main/java/org/openremote/agent/protocol/mqtt/AbstractMQTT_IOClient.javaindex e4028a3f4..076fe7f24 100644--- a/agent/src/main/java/org/openremote/agent/protocol/mqtt/AbstractMQTT_IOClient.java+++ b/agent/src/main/java/org/openremote/agent/protocol/mqtt/AbstractMQTT_IOClient.java@@ -261,7 +261,7 @@ public abstract class AbstractMQTT_IOClient<S> implements IOClient<MQTTMessage<SLOG.fine("Subscribed to topic '" + topic + "' on client '" + getClientUri() + "'");return true;} catch (Exception e) {- LOG.log(Level.WARNING, "Failed to subscribe to topic '" + topic + "' on client '" + getClientUri() + "'", e);+ LOG.log(Level.WARNING, "Failed to subscribe to topic '" + topic + "' on client '" + getClientUri() + "': " + e.getMessage());executorService.execute(() -> onSubscribeFailed(topic));}return false;@@ -366,7 +366,7 @@ public abstract class AbstractMQTT_IOClient<S> implements IOClient<MQTTMessage<ScompletableFutureSend.send().whenComplete((connAck, throwable) -> {if (throwable != null) {- LOG.log(Level.INFO, "Connection failed:" + getClientUri(), throwable);+ LOG.log(Level.INFO, "Connection failed:" + getClientUri(), throwable.getMessage());} else {synchronized (connected) {connected.set(true);diff --git a/agent/src/main/java/org/openremote/agent/protocol/simulator/SimulatorProtocol.java b/agent/src/main/java/org/openremote/agent/protocol/simulator/SimulatorProtocol.javaindex 75baf712b..8ee1e030d 100644--- a/agent/src/main/java/org/openremote/agent/protocol/simulator/SimulatorProtocol.java+++ b/agent/src/main/java/org/openremote/agent/protocol/simulator/SimulatorProtocol.java@@ -32,12 +32,12 @@ import org.openremote.model.syslog.SyslogCategory;import java.time.LocalDateTime;import java.time.temporal.ChronoField;import java.util.*;+import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ScheduledFuture;import java.util.concurrent.TimeUnit;import java.util.logging.Level;import java.util.logging.Logger;-import static org.openremote.container.concurrent.GlobalLock.withLock;import static org.openremote.model.syslog.SyslogCategory.PROTOCOL;public class SimulatorProtocol extends AbstractProtocol<SimulatorAgent, SimulatorAgentLink> {@@ -45,7 +45,7 @@ public class SimulatorProtocol extends AbstractProtocol<SimulatorAgent, Simulatoprivate static final Logger LOG = SyslogCategory.getLogger(PROTOCOL, SimulatorProtocol.class);public static final String PROTOCOL_DISPLAY_NAME = "Simulator";- protected final Map<AttributeRef, ScheduledFuture<?>> replayMap = new HashMap<>();+ protected final Map<AttributeRef, ScheduledFuture<?>> replayMap = new ConcurrentHashMap<>();public SimulatorProtocol(SimulatorAgent agent) {super(agent);@@ -68,6 +68,13 @@ public class SimulatorProtocol extends AbstractProtocol<SimulatorAgent, Simulato@Overrideprotected void doStop(Container container) throws Exception {+ replayMap.values().forEach(scheduledFuture -> {+ try {+ scheduledFuture.cancel(true);+ } catch (Exception ignored) {++ }+ });}@Override@@ -156,16 +163,14 @@ public class SimulatorProtocol extends AbstractProtocol<SimulatorAgent, SimulatoLOG.fine("Next update for asset " + attributeRef.getId() + " for attribute " + attributeRef.getName() + " in " + nextRunRelative + " second(s)");return executorService.schedule(() -> {- withLock(getProtocolName() + "::firingNextUpdate", () -> {- LOG.fine("Updating asset " + attributeRef.getId() + " for attribute " + attributeRef.getName() + " with value " + nextDatapoint.value.toString());- try {- updateLinkedAttribute(new AttributeState(attributeRef, nextDatapoint.value));- } catch (Exception e) {- LOG.log(Level.SEVERE, "Exception thrown when updating value: %s", e);- } finally {- replayMap.put(attributeRef, scheduleReplay(attributeRef, simulatorReplayDatapoints));- }- });+ LOG.fine("Updating asset " + attributeRef.getId() + " for attribute " + attributeRef.getName() + " with value " + nextDatapoint.value.toString());+ try {+ updateLinkedAttribute(new AttributeState(attributeRef, nextDatapoint.value));+ } catch (Exception e) {+ LOG.log(Level.SEVERE, "Exception thrown when updating value: %s", e);+ } finally {+ replayMap.put(attributeRef, scheduleReplay(attributeRef, simulatorReplayDatapoints));+ }}, nextRunRelative, TimeUnit.SECONDS);}}diff --git a/container/build.gradle b/container/build.gradleindex f6b87c692..0c9f2de46 100644--- a/container/build.gradle+++ b/container/build.gradle@@ -46,8 +46,10 @@ dependencies {api "org.apache.camel:camel-snmp:$camelVersion"api "org.apache.camel:camel-health:$camelVersion"api "org.apache.camel:camel-management:$camelVersion"- api("com.fasterxml.uuid:java-uuid-generator:$uuidGeneratorVersion")+ api "com.fasterxml.uuid:java-uuid-generator:$uuidGeneratorVersion"api "com.vladmihalcea:hibernate-types-60:$hibernateTypesVersion"+ api "org.apache.camel:camel-micrometer:$camelVersion"+ api "io.micrometer:micrometer-registry-prometheus:$micrometerVersion"}jar {diff --git a/container/src/main/java/org/openremote/container/Container.java b/container/src/main/java/org/openremote/container/Container.javaindex c5542af23..fb17985a8 100644--- a/container/src/main/java/org/openremote/container/Container.java+++ b/container/src/main/java/org/openremote/container/Container.java@@ -20,6 +20,12 @@package org.openremote.container;import com.fasterxml.jackson.databind.SerializationFeature;+import io.micrometer.core.instrument.Clock;+import io.micrometer.core.instrument.MeterRegistry;+import io.micrometer.core.instrument.Metrics;+import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics;+import io.micrometer.prometheus.PrometheusConfig;+import io.prometheus.client.CollectorRegistry;import org.openremote.container.concurrent.ContainerScheduledExecutor;import org.openremote.container.concurrent.ContainerThreads;import org.openremote.container.util.LogUtil;@@ -50,34 +56,13 @@ import static org.openremote.container.util.MapAccess.getInteger;*/public class Container implements org.openremote.model.Container {- protected static class NoShutdownScheduledExecutorService extends ContainerScheduledExecutor {-- public NoShutdownScheduledExecutorService(String name, int corePoolSize) {- super(name, corePoolSize);- }-- @Override- public void shutdown() {- throw new UnsupportedOperationException();- }-- @SuppressWarnings("NullableProblems")- @Override- public List<Runnable> shutdownNow() {- throw new UnsupportedOperationException();- }-- void doShutdownNow() {- super.shutdownNow();- }- }-public static final System.Logger LOG = System.getLogger(Container.class.getName());public static ScheduledExecutorService EXECUTOR_SERVICE;public static final String OR_SCHEDULED_TASKS_THREADS_MAX = "OR_SCHEDULED_TASKS_THREADS_MAX";public static final int OR_SCHEDULED_TASKS_THREADS_MAX_DEFAULT = Math.max(Runtime.getRuntime().availableProcessors(), 2);protected final Map<String, String> config = new HashMap<>();protected final boolean devMode;+ protected MeterRegistry meterRegistry;protected Thread waitingThread;protected final Map<Class<? extends ContainerService>, ContainerService> services = new LinkedHashMap<>();@@ -117,12 +102,24 @@ public class Container implements org.openremote.model.Container {ValueUtil.JSON.enable(SerializationFeature.INDENT_OUTPUT);}+ boolean metricsEnabled = getBoolean(getConfig(), OR_METRICS_ENABLED, OR_METRICS_ENABLED_DEFAULT);+ LOG.log(INFO, "Metrics enabled: " + metricsEnabled);++ if (metricsEnabled) {+ // TODO: Add a meter registry provider SPI to make this pluggable+ meterRegistry = new io.micrometer.prometheus.PrometheusMeterRegistry(PrometheusConfig.DEFAULT, io.prometheus.client.CollectorRegistry.defaultRegistry, Clock.SYSTEM);+ }+int scheduledTasksThreadsMax = getInteger(getConfig(),OR_SCHEDULED_TASKS_THREADS_MAX,OR_SCHEDULED_TASKS_THREADS_MAX_DEFAULT);- EXECUTOR_SERVICE = new NoShutdownScheduledExecutorService("Scheduled task", scheduledTasksThreadsMax);+ EXECUTOR_SERVICE = new ContainerScheduledExecutor("Scheduled task", scheduledTasksThreadsMax);++ if (meterRegistry != null) {+ EXECUTOR_SERVICE = ExecutorServiceMetrics.monitor(meterRegistry, EXECUTOR_SERVICE, "ContainerExecutorService");+ }// Any log handlers of the root logger that are container services must be registeredfor (Handler handler : Logger.getLogger("").getHandlers()) {@@ -179,22 +176,25 @@ public class Container implements org.openremote.model.Container {List<ContainerService> servicesToStop = Arrays.asList(getServices());Collections.reverse(servicesToStop);- try {- for (ContainerService service : servicesToStop) {- LOG.log(INFO, "Stopping service: " + service.getClass().getName());+ for (ContainerService service : servicesToStop) {+ LOG.log(INFO, "Stopping service: " + service.getClass().getName());+ try {service.stop(this);+ } catch (Exception e) {+ LOG.log(INFO, "Exception thrown whilst stopping service: " + service.getClass().getName(), e);}- } catch (Exception ex) {- throw new RuntimeException(ex);}try {LOG.log(INFO, "Cancelling scheduled tasks");- ((NoShutdownScheduledExecutorService) EXECUTOR_SERVICE).doShutdownNow();+ EXECUTOR_SERVICE.shutdown();} catch (Exception e) {LOG.log(WARNING, "Exception thrown whilst trying to stop scheduled tasks", e);}+ Metrics.globalRegistry.remove(meterRegistry);+ CollectorRegistry.defaultRegistry.clear();+ meterRegistry = null;waitingThread.interrupt();waitingThread = null;LOG.log(INFO, "<<< Runtime container stopped");@@ -234,6 +234,11 @@ public class Container implements org.openremote.model.Container {return getServices(type).size() > 0;}+ @Override+ public MeterRegistry getMeterRegistry() {+ return meterRegistry;+ }+/*** Get a service instance matching the specified type exactly, or if that yields* no result, try to get the first service instance that has a matching interface.diff --git a/container/src/main/java/org/openremote/container/concurrent/ContainerExecutor.java b/container/src/main/java/org/openremote/container/concurrent/ContainerExecutor.javaindex d500be994..b89ebca41 100644--- a/container/src/main/java/org/openremote/container/concurrent/ContainerExecutor.java+++ b/container/src/main/java/org/openremote/container/concurrent/ContainerExecutor.java@@ -24,15 +24,12 @@ import java.util.concurrent.*;import static org.openremote.container.concurrent.ContainerThreads.DEFAULT_REJECTED_EXECUTION_HANDLER;import static org.openremote.container.concurrent.ContainerThreads.logExceptionCause;+/**+ * Thread pool that adds logging for tasks that throw exceptions+ */public class ContainerExecutor extends ThreadPoolExecutor {- /**- * Creates an unbounded thread pool with {@link SynchronousQueue}, this is the same as- * {@link Executors#newCachedThreadPool}.- */- public ContainerExecutor(String name) {- this(name, 0, Integer.MAX_VALUE, 60L, -1);- }+ protected String name;/*** @param blockingQueueCapacity Set to <code>-1</code> if a {@link SynchronousQueue} should be used.@@ -42,23 +39,17 @@ public class ContainerExecutor extends ThreadPoolExecutor {int maximumPoolSize,long keepAliveSeconds,int blockingQueueCapacity) {- this(- new ContainerThreadFactory(name),- DEFAULT_REJECTED_EXECUTION_HANDLER,+ super(corePoolSize,maximumPoolSize,keepAliveSeconds,- blockingQueueCapacity == -1 ? new SynchronousQueue<>() : new ArrayBlockingQueue<>(blockingQueueCapacity)+ TimeUnit.SECONDS,+ blockingQueueCapacity == -1 ? new SynchronousQueue<>() : new ArrayBlockingQueue<>(blockingQueueCapacity),+ new ContainerThreadFactory(name),+ DEFAULT_REJECTED_EXECUTION_HANDLER);- }- public ContainerExecutor(ThreadFactory threadFactory,- RejectedExecutionHandler rejectedHandler,- int corePoolSize,- int maximumPoolSize,- long keepAliveSeconds,- BlockingQueue<Runnable> workQueue) {- super(corePoolSize, maximumPoolSize, keepAliveSeconds, TimeUnit.SECONDS, workQueue, threadFactory, rejectedHandler);+ this.name = name;}@Override@@ -66,4 +57,9 @@ public class ContainerExecutor extends ThreadPoolExecutor {super.afterExecute(runnable, throwable);logExceptionCause(runnable, throwable);}-}\ No newline at end of file++ @Override+ public String toString() {+ return name + ":" + super.toString();+ }+}diff --git a/container/src/main/java/org/openremote/container/concurrent/ContainerScheduledExecutor.java b/container/src/main/java/org/openremote/container/concurrent/ContainerScheduledExecutor.javaindex 1a7e927ea..43733691f 100644--- a/container/src/main/java/org/openremote/container/concurrent/ContainerScheduledExecutor.java+++ b/container/src/main/java/org/openremote/container/concurrent/ContainerScheduledExecutor.java@@ -26,6 +26,9 @@ import java.util.concurrent.ThreadFactory;import static org.openremote.container.concurrent.ContainerThreads.DEFAULT_REJECTED_EXECUTION_HANDLER;import static org.openremote.container.concurrent.ContainerThreads.logExceptionCause;+/**+ * Thread pool that adds logging for tasks that throw exceptions+ */public class ContainerScheduledExecutor extends ScheduledThreadPoolExecutor {public ContainerScheduledExecutor(String name, int corePoolSize) {@@ -42,4 +45,4 @@ public class ContainerScheduledExecutor extends ScheduledThreadPoolExecutor {logExceptionCause(runnable, throwable);}-}\ No newline at end of file+}diff --git a/container/src/main/java/org/openremote/container/concurrent/ContainerThreadFactory.java b/container/src/main/java/org/openremote/container/concurrent/ContainerThreadFactory.javaindex d5cf81ebc..637d98ccf 100644--- a/container/src/main/java/org/openremote/container/concurrent/ContainerThreadFactory.java+++ b/container/src/main/java/org/openremote/container/concurrent/ContainerThreadFactory.java@@ -39,6 +39,7 @@ public class ContainerThreadFactory implements ThreadFactory {group = Thread.currentThread().getThreadGroup();}+ @Overridepublic Thread newThread(Runnable r) {Thread t = new Thread(group, r, name + "-" + threadNumber.getAndIncrement(), 0);t.setDaemon(true);@@ -46,4 +47,4 @@ public class ContainerThreadFactory implements ThreadFactory {t.setPriority(Thread.NORM_PRIORITY);return t;}-}\ No newline at end of file+}diff --git a/container/src/main/java/org/openremote/container/concurrent/ContainerThreads.java b/container/src/main/java/org/openremote/container/concurrent/ContainerThreads.javaindex f84c9d9e3..1c3cdb83d 100644--- a/container/src/main/java/org/openremote/container/concurrent/ContainerThreads.java+++ b/container/src/main/java/org/openremote/container/concurrent/ContainerThreads.java@@ -33,7 +33,7 @@ public interface ContainerThreads {@Overridepublic void rejectedExecution(Runnable runnable, ThreadPoolExecutor threadPoolExecutor) {// Log and discard- LOG.info("Container thread pool '" + threadPoolExecutor + "' rejected execution of " + runnable.getClass());+ LOG.info("Container thread pool '" + threadPoolExecutor + "' rejected execution of " + runnable);super.rejectedExecution(runnable, threadPoolExecutor);}};diff --git a/container/src/main/java/org/openremote/container/concurrent/GlobalLock.java b/container/src/main/java/org/openremote/container/concurrent/GlobalLock.javadeleted file mode 100644index 65869806f..000000000--- a/container/src/main/java/org/openremote/container/concurrent/GlobalLock.java+++ /dev/null@@ -1,140 +0,0 @@-/*- * Copyright 2018, OpenRemote Inc.- *- * See the CONTRIBUTORS.txt file in the distribution for a- * full listing of individual contributors.- *- * This program is free software: you can redistribute it and/or modify- * it under the terms of the GNU Affero General Public License as- * published by the Free Software Foundation, either version 3 of the- * License, or (at your option) any later version.- *- * This program is distributed in the hope that it will be useful,- * but WITHOUT ANY WARRANTY; without even the implied warranty of- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the- * GNU Affero General Public License for more details.- *- * You should have received a copy of the GNU Affero General Public License- * along with this program. If not, see <http://www.gnu.org/licenses/>.- */-package org.openremote.container.concurrent;--import java.util.Optional;-import java.util.concurrent.TimeUnit;-import java.util.concurrent.locks.ReentrantLock;-import java.util.function.Supplier;-import java.util.logging.Logger;--import static java.util.logging.Level.INFO;--/**- * A global reentrant exclusive lock, use convenience methods {@link #withLock} and {@link #withLockReturning}.- */-public class GlobalLock {-- protected static final int LOCK_WARNING_MILLIS = 10000;- private static final Logger LOG = Logger.getLogger(GlobalLock.class.getName());-- /**- * At least getOwner() is protected and not private...- */- static class CustomReentrantLock extends ReentrantLock {-- String info;-- public CustomReentrantLock() {- super(true);- }-- String owner() {- Thread lockOwner;- if ((lockOwner = super.getOwner()) != null) {- return lockOwner.getName() + " executing " + info;- }- return "Unknown executing " + info;- }-- String getOwnerThreadName() {- return Optional.ofNullable(super.getOwner()).map(Thread::getName).orElse(null);- }-- public boolean tryLock(String info, long timeout, TimeUnit unit) throws InterruptedException {- boolean result = super.tryLock(timeout, unit);- if (result) {- this.info = info;- }- return result;- }-- @Override- public void unlock() {- super.unlock();- info = null;- }- }-- // Provides exclusive access to shared state- static protected final CustomReentrantLock lock = new CustomReentrantLock();-- protected GlobalLock() {- }-- /**- * @return Defaults to 10 seconds.- */- static public int getLockTimeoutMillis() {- return 30000;- }-- /**- * Obtain the lock within {@link #getLockTimeoutMillis()} or throw {@link IllegalStateException}.- *- * @param info An informal text that is printed in log messages.- * @param runnable The guarded code to execute while holding the lock.- */- static public void withLock(String info, Runnable runnable) {- withLockReturning(info, () -> {- runnable.run();- return null;- });- }-- /**- * Obtain the lock within {@link #getLockTimeoutMillis()} or throw {@link IllegalStateException}.- *- * @param info An informal text that is printed in log messages.- * @param supplier The guarded code to execute while holding the lock.- */- static public <R> R withLockReturning(String info, Supplier<R> supplier) {- try {- long startTime = System.currentTimeMillis();- if (lock.tryLock(info, getLockTimeoutMillis(), TimeUnit.MILLISECONDS)) {- LOG.finest("+ Acquired lock (count: " + lock.getHoldCount() + "): " + info);- String threadName = lock.getOwnerThreadName();- try {- return supplier.get();- } finally {- LOG.finest("- Releasing lock (count: " + lock.getHoldCount() + "): " + info);- if (!lock.isHeldByCurrentThread()) {- LOG.severe("Lock is held by another thread, ensure the same thread acquires and releases the lock!");- } else {- lock.unlock();- }- long duration = System.currentTimeMillis() - startTime;- if (duration > LOCK_WARNING_MILLIS) {- LOG.warning("Lock duration was above warning threshold for thread (" + threadName + "): " + duration + "ms");- } else {- LOG.fine("Lock duration for thread (" + threadName + "): " + duration + "ms");- }- }- } else {- throw new IllegalStateException(- "Could not acquire lock owned by " + lock.owner() + " after waiting " + getLockTimeoutMillis() + "ms: " + Thread.currentThread().getName() + " executing " + info- );- }- } catch (InterruptedException ex) {- LOG.log(INFO, "Interrupted while waiting for lock: " + info);- return null;- }- }-}diff --git a/container/src/main/java/org/openremote/container/message/MessageBrokerService.java b/container/src/main/java/org/openremote/container/message/MessageBrokerService.javaindex ca304ce61..e9a9957c3 100644--- a/container/src/main/java/org/openremote/container/message/MessageBrokerService.java+++ b/container/src/main/java/org/openremote/container/message/MessageBrokerService.java@@ -19,9 +19,13 @@*/package org.openremote.container.message;+import io.micrometer.core.instrument.MeterRegistry;+import io.micrometer.core.instrument.Tags;+import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics;import org.apache.camel.FluentProducerTemplate;import org.apache.camel.ProducerTemplate;import org.apache.camel.builder.DefaultErrorHandlerBuilder;+import org.apache.camel.component.micrometer.spi.TimedScheduledExecutorService;import org.apache.camel.component.snmp.SnmpComponent;import org.apache.camel.health.HealthCheckRegistry;import org.apache.camel.impl.DefaultCamelContext;@@ -47,8 +51,11 @@ import java.util.Map;import java.util.concurrent.ExecutorService;import java.util.concurrent.ScheduledExecutorService;import java.util.concurrent.ThreadFactory;+import java.util.concurrent.atomic.AtomicLong;import java.util.logging.Logger;+import static org.openremote.container.util.MapAccess.getString;+public class MessageBrokerService implements ContainerService {private static final Logger LOG = Logger.getLogger(MessageBrokerService.class.getName());@@ -67,33 +74,66 @@ public class MessageBrokerService implements ContainerService {@Overridepublic void init(Container container) throws Exception {+ MeterRegistry meterRegistry = container.getMeterRegistry();final ExecutorServiceManager executorServiceManager = context.getExecutorServiceManager();- executorServiceManager.setThreadNamePattern("#counter# #name#");- executorServiceManager.setThreadPoolFactory(new ThreadPoolFactory() {++ // Not using InstrumentedThreadPoolFactory directly as it only uses the ThreadPoolProfile ID for naming+ ThreadPoolFactory threadPoolFactory = new ThreadPoolFactory() {+ private static final AtomicLong COUNTER = new AtomicLong();+@Overridepublic ExecutorService newCachedThreadPool(ThreadFactory threadFactory) {- // This is an unlimited pool used probably only be multicast aggregation- return new ContainerExecutor(getExecutorName("MessagingPool", threadFactory), 1, Integer.MAX_VALUE, 10, -1);+ // This is an unlimited pool used probably only by multicast aggregation+ ExecutorService executorService = new ContainerExecutor(+ getExecutorName("CachedPool", threadFactory),+ 1,+ Integer.MAX_VALUE,+ 10,+ -1);++// if (meterRegistry != null) {+// executorService = ExecutorServiceMetrics.monitor(meterRegistry, executorService, name("instrumented-delegate-"));+// }++ return executorService;}@Overridepublic ExecutorService newThreadPool(ThreadPoolProfile profile, ThreadFactory threadFactory) {// This pool is used by SEDA consumers, so the endpoint parameters define the pool and queue sizes- return new ContainerExecutor(- getExecutorName("Messaging", threadFactory),+ ExecutorService executorService = new ContainerExecutor(+ getExecutorName("Pool", threadFactory),profile.getPoolSize(),profile.getMaxPoolSize(),profile.getKeepAliveTime(),profile.getMaxQueueSize());++// Disabled as not very useful for SEDA components+// if (meterRegistry != null) {+// String name = getExecutorName("Pool", threadFactory);+// name = "Pool".equals(name) ? profile.getId() : name;+// executorService = ExecutorServiceMetrics.monitor(meterRegistry, executorService, name(name));+// }++ return executorService;}@Overridepublic ScheduledExecutorService newScheduledThreadPool(ThreadPoolProfile profile, ThreadFactory threadFactory) {- return new ContainerScheduledExecutor(- getExecutorName("MessagingTasks", threadFactory),+ ScheduledExecutorService scheduledExecutorService = new ContainerScheduledExecutor(+ getExecutorName("ScheduledPool", threadFactory),profile.getPoolSize());++// Disabled as not very useful for SEDA components+// if (meterRegistry != null) {+// String name = getExecutorName("", threadFactory);+// name = "ScheduledPool".equals(name) ? profile.getId() : name;+// scheduledExecutorService = new TimedScheduledExecutorService(meterRegistry, scheduledExecutorService, name(name), Tags.empty());+// }++ return scheduledExecutorService;}protected String getExecutorName(String name, ThreadFactory threadFactory) {@@ -106,7 +146,14 @@ public class MessageBrokerService implements ContainerService {}return name;}- });++ private String name(String prefix) {+ return prefix + COUNTER.incrementAndGet();+ }+ };++ executorServiceManager.setThreadNamePattern("#counter# #name#");+ executorServiceManager.setThreadPoolFactory(threadPoolFactory);// TODO might need this for errorhandler?context.setAllowUseOriginalMessage(false);@@ -124,7 +171,7 @@ public class MessageBrokerService implements ContainerService {context.setExtension(HealthCheckRegistry.class, checkRegistry);// Force a quick shutdown of routes with in-flight exchanges- context.getShutdownStrategy().setTimeout(1);+ context.getShutdownStrategy().setTimeout(5);context.getShutdownStrategy().setSuppressLoggingOnTimeout(true);context.setStreamCaching(true);diff --git a/container/src/main/java/org/openremote/container/persistence/PersistenceHealthStatusProvider.java b/container/src/main/java/org/openremote/container/persistence/PersistenceHealthStatusProvider.javaindex 14cad6545..80e69d7e7 100644--- a/container/src/main/java/org/openremote/container/persistence/PersistenceHealthStatusProvider.java+++ b/container/src/main/java/org/openremote/container/persistence/PersistenceHealthStatusProvider.java@@ -36,7 +36,7 @@ import java.util.logging.Logger;import static org.openremote.container.persistence.Database.PROPERTY_POOL_NAME;-public class PersistenceHealthStatusProvider implements HealthStatusProvider, ContainerService {+public class PersistenceHealthStatusProvider implements HealthStatusProvider {private static final Logger LOG = Logger.getLogger(PersistenceHealthStatusProvider.class.getName());public static final String NAME = "db";diff --git a/docker-compose.yml b/docker-compose.ymlindex 61a972c81..0d0b642fb 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -21,9 +21,10 @@ services:manager:condition: service_healthyports:- - "80:80"+ - "80:80" # Needed for SSL generation using letsencrypt- "${OR_SSL_PORT:-443}:443"- "8883:8883"+ #- "127.0.0.1:8404:8404" # Localhost metrics accessvolumes:- proxy-data:/deploymentenvironment:@@ -61,6 +62,8 @@ services:depends_on:keycloak:condition: service_healthy+ #ports:+ # - "127.0.0.1:8405:8404" # Localhost metrics accessenvironment:OR_SETUP_TYPE:OR_ADMIN_PASSWORD:diff --git a/gradle.properties b/gradle.propertiesindex 0caf5cb0b..1faec5c7f 100644--- a/gradle.properties+++ b/gradle.properties@@ -24,7 +24,7 @@ junitVersion = 4.13.2spockVersion = 2.0-groovy-3.0byteBuddyVersion = 1.11.0objenesisVersion = 3.2-camelVersion = 3.19.0+camelVersion = 3.20.6sunriseSunsetCalculatorVersion = 1.2easyRulesVersion = 4.1.0quartzVersion= 2.3.2@@ -56,6 +56,8 @@ failsafeVersion=3.1.0suncalcVersion=3.5snapshotsRepoUrl=https://s01.oss.sonatype.org/content/repositories/snapshots/releasesRepoUrl=https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/-activeMQVersion = 2.26.0+activeMQVersion = 2.29.0jbossLoggingManagerVersion=2.1.19.FinalgreenmailVersion=2.0.0+prometheusVersion=0.16.0+micrometerVersion=1.11.0diff --git a/manager/Dockerfile b/manager/Dockerfileindex b16eb1f22..a175ba940 100644--- a/manager/Dockerfile+++ b/manager/Dockerfile@@ -61,6 +61,9 @@ ENV OR_JAVA_OPTS ${OR_JAVA_OPTS:--Xms500m -Xmx2g \-Xlog:all=warning:stdout:uptime,level,tags \-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/dump.hprof}+# OpenMetrics _created series breaks cloudwatch prometheus scraper+ENV PROMETHEUS_DISABLE_CREATED_SERIES ${PROMETHEUS_DISABLE_CREATED_SERIES:-true}+RUN mkdir -p /deployment/manager/extensionsWORKDIR /opt/appdiff --git a/manager/build.gradle b/manager/build.gradleindex 7e75d9269..4af65e6f5 100644--- a/manager/build.gradle+++ b/manager/build.gradle@@ -50,8 +50,8 @@ dependencies {}implementation "org.shredzone.commons:commons-suncalc:$suncalcVersion"-implementation "org.slf4j:slf4j-api:$slf4jVersion"+ implementation "io.prometheus:simpleclient_httpserver:$prometheusVersion"}jar {diff --git a/manager/src/main/java/org/openremote/manager/agent/AgentHealthStatusProvider.java b/manager/src/main/java/org/openremote/manager/agent/AgentHealthStatusProvider.javaindex ef0bb3a3a..5e039541b 100644--- a/manager/src/main/java/org/openremote/manager/agent/AgentHealthStatusProvider.java+++ b/manager/src/main/java/org/openremote/manager/agent/AgentHealthStatusProvider.java@@ -29,7 +29,7 @@ import org.openremote.model.util.ValueUtil;import java.util.concurrent.atomic.AtomicInteger;-public class AgentHealthStatusProvider implements HealthStatusProvider, ContainerService {+public class AgentHealthStatusProvider implements HealthStatusProvider {public static final String NAME = "agents";protected AgentService agentService;@@ -103,7 +103,7 @@ public class AgentHealthStatusProvider implements HealthStatusProvider, ContaineobjectValue.set(agent.getId(), agentValue);}- objectValue.put("totalAgents", agentService.agentMap.size());+ objectValue.put("totalAgents", agentService.getAgents().size());objectValue.put("connectedAgents", connectedCount.get());objectValue.put("errorAgents", errorCount.get());objectValue.put("disabledAgents", disabledCount.get());diff --git a/manager/src/main/java/org/openremote/manager/agent/AgentService.java b/manager/src/main/java/org/openremote/manager/agent/AgentService.javaindex b78419243..df6b092d0 100644--- a/manager/src/main/java/org/openremote/manager/agent/AgentService.java+++ b/manager/src/main/java/org/openremote/manager/agent/AgentService.java@@ -19,10 +19,10 @@*/package org.openremote.manager.agent;+import jakarta.persistence.EntityManager;import org.apache.camel.builder.RouteBuilder;import org.openremote.agent.protocol.ProtocolAssetService;import org.openremote.container.message.MessageBrokerService;-import org.openremote.model.PersistenceEvent;import org.openremote.container.timer.TimerService;import org.openremote.manager.asset.AssetProcessingException;import org.openremote.manager.asset.AssetProcessingService;@@ -34,6 +34,7 @@ import org.openremote.manager.security.ManagerIdentityService;import org.openremote.manager.web.ManagerWebService;import org.openremote.model.Container;import org.openremote.model.ContainerService;+import org.openremote.model.PersistenceEvent;import org.openremote.model.asset.Asset;import org.openremote.model.asset.AssetTreeNode;import org.openremote.model.asset.agent.Agent;@@ -53,9 +54,8 @@ import org.openremote.model.query.filter.StringPredicate;import org.openremote.model.util.Pair;import org.openremote.model.util.TextUtil;-import jakarta.persistence.EntityManager;import java.util.*;-import java.util.concurrent.ConcurrentMap;+import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.Future;import java.util.concurrent.ScheduledExecutorService;import java.util.function.Consumer;@@ -67,11 +67,9 @@ import java.util.stream.Stream;import static java.util.stream.Collectors.mapping;import static java.util.stream.Collectors.toList;-import static org.openremote.container.concurrent.GlobalLock.withLock;-import static org.openremote.container.concurrent.GlobalLock.withLockReturning;import static org.openremote.container.persistence.PersistenceService.PERSISTENCE_TOPIC;import static org.openremote.container.persistence.PersistenceService.isPersistenceEventForEntityType;-import static org.openremote.manager.asset.AssetProcessingService.ASSET_QUEUE;+import static org.openremote.manager.asset.AssetProcessingService.ATTRIBUTE_EVENT_QUEUE;import static org.openremote.manager.gateway.GatewayService.isNotForGateway;import static org.openremote.model.asset.agent.Protocol.ACTUATOR_TOPIC;import static org.openremote.model.asset.agent.Protocol.SENSOR_QUEUE;@@ -97,10 +95,10 @@ public class AgentService extends RouteBuilder implements ContainerService, Asseprotected ClientEventService clientEventService;protected GatewayService gatewayService;protected ScheduledExecutorService executorService;- protected ConcurrentMap<String, Agent<?, ?, ?>> agentMap;- protected final Map<String, Future<Void>> agentDiscoveryImportFutureMap = new HashMap<>();- protected final Map<String, Protocol<?>> protocolInstanceMap = new HashMap<>();- protected final Map<String, List<Consumer<PersistenceEvent<Asset<?>>>>> childAssetSubscriptions = new HashMap<>();+ protected Map<String, Agent<?, ?, ?>> agentMap;+ protected final Map<String, Future<Void>> agentDiscoveryImportFutureMap = new ConcurrentHashMap<>();+ protected final Map<String, Protocol<?>> protocolInstanceMap = new ConcurrentHashMap<>();+ protected final Map<String, Set<Consumer<PersistenceEvent<Asset<?>>>>> childAssetSubscriptions = new ConcurrentHashMap<>();protected boolean initDone;protected Container container;@@ -151,9 +149,11 @@ public class AgentService extends RouteBuilder implements ContainerService, Asse@Overridepublic void stop(Container container) throws Exception {- List<Agent<?,?,?>> agents = new ArrayList<>(agentMap.values());- agents.forEach(agent -> this.stopAgent(agent.getId()));- agentMap.clear();+ if (agentMap != null) {+ List<Agent<?, ?, ?>> agents = new ArrayList<>(agentMap.values());+ agents.forEach(agent -> this.stopAgent(agent.getId()));+ agentMap.clear();+ }protocolInstanceMap.clear();}@@ -161,7 +161,7 @@ public class AgentService extends RouteBuilder implements ContainerService, Asse@Overridepublic void configure() throws Exception {from(PERSISTENCE_TOPIC)- .routeId("AgentPersistenceChanges")+ .routeId("Persistence-Agent").filter(isPersistenceEventForEntityType(Asset.class)).filter(isNotForGateway(gatewayService)).process(exchange -> {@@ -177,10 +177,10 @@ public class AgentService extends RouteBuilder implements ContainerService, Asse// A protocol wants to write a new sensor valuefrom(SENSOR_QUEUE)- .routeId("FromSensorUpdates")+ .routeId("ProtocolOutbound").filter(body().isInstanceOf(AttributeEvent.class)).setHeader(HEADER_SOURCE, () -> SENSOR)- .to(ASSET_QUEUE);+ .to(ATTRIBUTE_EVENT_QUEUE);}@Override@@ -383,90 +383,86 @@ public class AgentService extends RouteBuilder implements ContainerService, Asse}protected void startAgent(Agent<?,?,?> agent) {- withLock(getClass().getSimpleName() + "::startAgent", () -> {- Protocol<?> protocol = null;+ Protocol<?> protocol = null;- try {- protocol = agent.getProtocolInstance();- protocolInstanceMap.put(agent.getId(), protocol);-- LOG.fine("Starting protocol instance: " + protocol);- protocol.start(container);- LOG.fine("Started protocol instance:" + protocol);-- LOG.finest("Linking attributes to protocol instance: " + protocol);-- // Get all assets that have attributes with agent link meta for this agent- List<Asset<?>> assets = assetStorageService.findAll(- new AssetQuery()- .attributes(- new AttributePredicate().meta(- new NameValuePredicate(AGENT_LINK, new StringPredicate(agent.getId()), false, new NameValuePredicate.Path("id"))- )+ try {+ protocol = agent.getProtocolInstance();++ LOG.fine("Starting protocol instance: " + protocol);+ protocol.start(container);+ protocolInstanceMap.put(agent.getId(), protocol);+ LOG.fine("Started protocol instance:" + protocol);++ LOG.finest("Linking attributes to protocol instance: " + protocol);++ // Get all assets that have attributes with agent link meta for this agent+ List<Asset<?>> assets = assetStorageService.findAll(+ new AssetQuery()+ .attributes(+ new AttributePredicate().meta(+ new NameValuePredicate(AGENT_LINK, new StringPredicate(agent.getId()), false, new NameValuePredicate.Path("id")))- );-- LOG.finest("Found '" + assets.size() + "' asset(s) with attributes linked to this protocol instance: " + protocol);-- assets.forEach(- asset ->- getGroupedAgentLinkAttributes(- asset.getAttributes().stream(),- assetAttribute -> assetAttribute.getMetaValue(AGENT_LINK)- .map(agentLink -> agentLink.getId().equals(agent.getId()))- .orElse(false)- ).forEach((agnt, attributes) -> linkAttributes(agnt, asset.getId(), attributes))- );- } catch (Exception e) {- if (protocol != null) {- try {- protocol.stop(container);- } catch (Exception ignored) {- }+ )+ );++ LOG.finest("Found '" + assets.size() + "' asset(s) with attributes linked to this protocol instance: " + protocol);++ assets.forEach(+ asset ->+ getGroupedAgentLinkAttributes(+ asset.getAttributes().stream(),+ assetAttribute -> assetAttribute.getMetaValue(AGENT_LINK)+ .map(agentLink -> agentLink.getId().equals(agent.getId()))+ .orElse(false)+ ).forEach((agnt, attributes) -> linkAttributes(agnt, asset.getId(), attributes))+ );+ } catch (Exception e) {+ if (protocol != null) {+ try {+ protocol.stop(container);+ } catch (Exception ignored) {}- protocolInstanceMap.remove(agent.getId());- LOG.log(Level.SEVERE, "Failed to start protocol instance for agent: " + agent, e);- sendAttributeEvent(new AttributeEvent(agent.getId(), Agent.STATUS.getName(), ConnectionStatus.ERROR));}- });+ protocolInstanceMap.remove(agent.getId());+ LOG.log(Level.SEVERE, "Failed to start protocol instance for agent: " + agent, e);+ sendAttributeEvent(new AttributeEvent(agent.getId(), Agent.STATUS.getName(), ConnectionStatus.ERROR));+ }}protected void stopAgent(String agentId) {- withLock(getClass().getSimpleName() + "::stopAgent", () -> {- Protocol<?> protocol = protocolInstanceMap.get(agentId);+ Protocol<?> protocol = protocolInstanceMap.get(agentId);- if (protocol == null) {- return;- }+ if (protocol == null) {+ return;+ }- Map<String, List<Attribute<?>>> groupedAttributes = protocol.getLinkedAttributes().entrySet().stream().collect(- Collectors.groupingBy(entry -> entry.getKey().getId(), mapping(Map.Entry::getValue, toList()))- );+ Map<String, List<Attribute<?>>> groupedAttributes = protocol.getLinkedAttributes().entrySet().stream().collect(+ Collectors.groupingBy(entry -> entry.getKey().getId(), mapping(Map.Entry::getValue, toList()))+ );- groupedAttributes.forEach((assetId, linkedAttributes) -> unlinkAttributes(agentId, assetId, linkedAttributes));+ groupedAttributes.forEach((assetId, linkedAttributes) -> unlinkAttributes(agentId, assetId, linkedAttributes));- // Stop the protocol instance- try {- protocol.stop(container);- } catch (Exception e) {- LOG.log(Level.SEVERE, "Protocol instance threw an exception whilst being stopped", e);- }+ // Stop the protocol instance+ try {+ protocol.stop(container);+ } catch (Exception e) {+ LOG.log(Level.SEVERE, "Protocol instance threw an exception whilst being stopped", e);+ }- // Remove child asset subscriptions for this agent- childAssetSubscriptions.remove(agentId);- protocolInstanceMap.remove(agentId);- });+ // Remove child asset subscriptions for this agent+ childAssetSubscriptions.remove(agentId);+ protocolInstanceMap.remove(agentId);}protected void linkAttributes(Agent<?,?,?> agent, String assetId, Collection<Attribute<?>> attributes) {- withLock(getClass().getSimpleName() + "::linkAttributes", () -> {- Protocol<?> protocol = getProtocolInstance(agent.getId());+ final Protocol<?> protocol = getProtocolInstance(agent.getId());- if (protocol == null) {- return;- }+ if (protocol == null) {+ return;+ }- LOG.fine("Linking asset '" + assetId + "' attributes linked to protocol: assetId=" + assetId + ", attributes=" + attributes.size() + ", protocol=" + protocol);+ synchronized (protocol) {+ LOG.fine("Linking asset '" + assetId + "' attributes linked to protocol: assetId=" + assetId + ", attributes=" + attributes.size() + ", protocol=" + protocol);attributes.forEach(attribute -> {AttributeRef attributeRef = new AttributeRef(assetId, attribute.getName());@@ -479,18 +475,18 @@ public class AgentService extends RouteBuilder implements ContainerService, AsseLOG.log(Level.SEVERE, "Failed to link attribute '" + attributeRef + "' to protocol: " + protocol, ex);}});- });+ }}protected void unlinkAttributes(String agentId, String assetId, List<Attribute<?>> attributes) {- withLock(getClass().getSimpleName() + "::unlinkAttributes", () -> {- Protocol<?> protocol = getProtocolInstance(agentId);+ final Protocol<?> protocol = getProtocolInstance(agentId);- if (protocol == null) {- return;- }+ if (protocol == null) {+ return;+ }- LOG.fine("Unlinking asset '" + assetId + "' attributes linked to protocol: assetId=" + assetId + ", attributes=" + attributes.size() + ", protocol=" + protocol);+ synchronized (protocol) {+ LOG.fine("Unlinking asset '" + assetId + "' attributes linked to protocol: assetId=" + assetId + ", attributes=" + attributes.size() + ", protocol=" + protocol);attributes.forEach(attribute -> {try {@@ -503,7 +499,7 @@ public class AgentService extends RouteBuilder implements ContainerService, AsseLOG.log(Level.SEVERE, "Ignoring error on unlinking attribute '" + attribute + "' from protocol: " + protocol, ex);}});- });+ }}/**@@ -549,20 +545,17 @@ public class AgentService extends RouteBuilder implements ContainerService, Assereturn false;}- Boolean result = withLockReturning(getClass().getSimpleName() + "::processAssetUpdate", () ->- attribute.getMetaValue(AGENT_LINK)- .map(agentLink -> {- LOG.finest("Attribute write for agent linked attribute: agent=" + agentLink.getId() + ", asset=" + asset.getId() + ", attribute=" + attribute.getName());-- messageBrokerService.getFluentProducerTemplate()- .withBody(attributeEvent)- .withHeader(Protocol.ACTUATOR_TOPIC_TARGET_PROTOCOL, getProtocolInstance(agentLink.getId()))- .to(ACTUATOR_TOPIC)- .asyncSend();- return true; // Processing complete, skip other processors- }).orElse(false) // This is a regular attribute so allow the processing to continue- );- return result != null ? result : false;+ return attribute.getMetaValue(AGENT_LINK)+ .map(agentLink -> {+ LOG.finest("Attribute write for agent linked attribute: agent=" + agentLink.getId() + ", asset=" + asset.getId() + ", attribute=" + attribute.getName());+ // TODO: Priority remove actuator topic and call protocol instance directly+ messageBrokerService.getFluentProducerTemplate()+ .withBody(attributeEvent)+ .withHeader(Protocol.ACTUATOR_TOPIC_TARGET_PROTOCOL, getProtocolInstance(agentLink.getId()))+ .to(ACTUATOR_TOPIC)+ .asyncSend();+ return true; // Processing complete, skip other processors+ }).orElse(false); // This is a regular attribute so allow the processing to continue}/**@@ -649,35 +642,29 @@ public class AgentService extends RouteBuilder implements ContainerService, Assereturn;}- withLock(getClass().getSimpleName() + "::subscribeChildAssetChange", () -> {- List<Consumer<PersistenceEvent<Asset<?>>>> consumerList = childAssetSubscriptions- .computeIfAbsent(agentId, (id) -> new ArrayList<>());- if (!consumerList.contains(assetChangeConsumer)) {- consumerList.add(assetChangeConsumer);- }- });+ Set<Consumer<PersistenceEvent<Asset<?>>>> consumers = childAssetSubscriptions+ .computeIfAbsent(agentId, (id) -> Collections.synchronizedSet(new HashSet<>()));+ consumers.add(assetChangeConsumer);}@Overridepublic void unsubscribeChildAssetChange(String agentId, Consumer<PersistenceEvent<Asset<?>>> assetChangeConsumer) {- withLock(getClass().getSimpleName() + "::unsubscribeChildAssetChange", () ->- childAssetSubscriptions.computeIfPresent(agentId, (id, consumerList) -> {- consumerList.remove(assetChangeConsumer);- return consumerList.isEmpty() ? null : consumerList;- }));+ childAssetSubscriptions.computeIfPresent(agentId, (id, consumers) -> {+ consumers.remove(assetChangeConsumer);+ return consumers.isEmpty() ? null : consumers;+ });}protected void notifyChildAssetChange(String agentId, PersistenceEvent<Asset<?>> assetPersistenceEvent) {- withLock(getClass().getSimpleName() + "::notifyChildAssetChange", () ->- childAssetSubscriptions.computeIfPresent(agentId, (id, consumerList) -> {- LOG.finest("Notifying child asset change consumers of change to agent child asset: Agent ID=" + id + ", Asset<?> ID=" + assetPersistenceEvent.getEntity().getId());- try {- consumerList.forEach(consumer -> consumer.accept(assetPersistenceEvent));- } catch (Exception e) {- LOG.log(Level.WARNING, "Child asset change consumer threw an exception: Agent ID=" + id + ", Asset<?> ID=" + assetPersistenceEvent.getEntity().getId(), e);- }- return consumerList;- }));+ childAssetSubscriptions.computeIfPresent(agentId, (id, consumers) -> {+ LOG.finest("Notifying child asset change consumers of change to agent child asset: Agent ID=" + id + ", Asset<?> ID=" + assetPersistenceEvent.getEntity().getId());+ try {+ consumers.forEach(consumer -> consumer.accept(assetPersistenceEvent));+ } catch (Exception e) {+ LOG.log(Level.WARNING, "Child asset change consumer threw an exception: Agent ID=" + id + ", Asset<?> ID=" + assetPersistenceEvent.getEntity().getId(), e);+ }+ return consumers;+ });}public boolean isProtocolAssetDiscoveryOrImportRunning(String agentId) {diff --git a/manager/src/main/java/org/openremote/manager/asset/AssetProcessingService.java b/manager/src/main/java/org/openremote/manager/asset/AssetProcessingService.javaindex 2419e72d1..91305924f 100644--- a/manager/src/main/java/org/openremote/manager/asset/AssetProcessingService.java+++ b/manager/src/main/java/org/openremote/manager/asset/AssetProcessingService.java@@ -19,6 +19,7 @@*/package org.openremote.manager.asset;+import jakarta.persistence.EntityManager;import org.apache.camel.Exchange;import org.apache.camel.Processor;import org.apache.camel.builder.RouteBuilder;@@ -46,14 +47,11 @@ import org.openremote.model.util.ValueUtil;import org.openremote.model.value.MetaItemType;import org.openremote.model.value.ValueType;-import jakarta.persistence.EntityManager;import java.util.*;import java.util.function.Supplier;import java.util.logging.Level;import java.util.logging.Logger;-import static org.openremote.container.concurrent.GlobalLock.withLock;-import static org.openremote.manager.event.ClientEventService.CLIENT_EVENT_TOPIC;import static org.openremote.model.attribute.AttributeEvent.HEADER_SOURCE;import static org.openremote.model.attribute.AttributeEvent.Source.*;import static org.openremote.model.attribute.AttributeWriteFailure.*;@@ -65,7 +63,7 @@ import static org.openremote.model.attribute.AttributeWriteFailure.*;* <dd><p>Client events published through event bus or sent by web service. These exchanges must contain an {@link AuthContext}* header named {@link Constants#AUTH_CONTEXT}.</dd>* <dt>{@link Source#INTERNAL}</dt>- * <dd><p>Events sent to {@link #ASSET_QUEUE} or through {@link #sendAttributeEvent} convenience method by processors.</dd>+ * <dd><p>Events sent to {@link #ATTRIBUTE_EVENT_QUEUE} or through {@link #sendAttributeEvent} convenience method by processors.</dd>* <dt>{@link Source#SENSOR}</dt>* <dd><p>Protocol sensor updates sent to {@link Protocol#SENSOR_QUEUE}.</dd>* </dl>@@ -120,9 +118,8 @@ import static org.openremote.model.attribute.AttributeWriteFailure.*;public class AssetProcessingService extends RouteBuilder implements ContainerService {public static final int PRIORITY = AssetStorageService.PRIORITY + 1000;- // TODO: Some of these options should be configurable depending on expected load etc.- // Message topic for communicating individual asset attribute changes- public static final String ASSET_QUEUE = "seda://AssetQueue?waitForTaskToComplete=IfReplyExpected&timeout=10000&purgeWhenStopping=true&discardIfNoConsumers=false&size=25000";+ // Single threaded attribute event queue (event order has to be maintained)+ public static final String ATTRIBUTE_EVENT_QUEUE = "seda://AttributeEventQueue?waitForTaskToComplete=IfReplyExpected&timeout=10000&purgeWhenStopping=true&discardIfNoConsumers=false&size=25000";private static final Logger LOG = Logger.getLogger(AssetProcessingService.class.getName());final protected List<AssetUpdateProcessor> processors = new ArrayList<>();protected TimerService timerService;@@ -275,25 +272,16 @@ public class AssetProcessingService extends RouteBuilder implements ContainerSer@Overridepublic void start(Container container) throws Exception {-}@Overridepublic void stop(Container container) throws Exception {-}@SuppressWarnings("rawtypes")@Overridepublic void configure() throws Exception {- // A client wants to write attribute state through event bus- from(CLIENT_EVENT_TOPIC)- .routeId("FromClientUpdates")- .filter(body().isInstanceOf(AttributeEvent.class))- .setHeader(HEADER_SOURCE, () -> CLIENT)- .to(ASSET_QUEUE);-// Process attribute events// TODO: Make SENDER much more granular (switch to microservices with RBAC)// TODO: Unify logging of attribute updates@@ -311,16 +299,17 @@ public class AssetProcessingService extends RouteBuilder implements ContainerSer- Replace at-most-once ClientEventService with at-least-once capable, embeddable message broker/protocol- See pseudocode here: http://activemq.apache.org/should-i-use-xa.html*/- from(ASSET_QUEUE)- .routeId("AssetQueueProcessor")- .filter(body().isInstanceOf(AttributeEvent.class))+ from(ATTRIBUTE_EVENT_QUEUE)+ .routeId("AttributeEventProcessor").doTry()- // Lock the global context, we can only process attribute events when the- // context isn't locked. Agent- and RulesService lock the context while protocols- // or rulesets are modified.- .process(exchange -> withLock(getClass().getSimpleName() + "::processFromAssetQueue", () -> {-+ .process(exchange -> {AttributeEvent event = exchange.getIn().getBody(AttributeEvent.class);++ // Set timestamp if not set+ if (event.getTimestamp() <= 0) {+ event.setTimestamp(timerService.getCurrentTimeMillis());+ }+if (LOG.isLoggable(Level.FINEST)) {LOG.finest("Processing: " + event);}@@ -430,14 +419,14 @@ public class AssetProcessingService extends RouteBuilder implements ContainerSer// Push through all processorsprocessAssetUpdate(em, asset, updatedAttribute, source);});- }))+ }).endDoTry().doCatch(AssetProcessingException.class).process(handleAssetProcessingException(LOG));}/**- * Send internal attribute change events into the {@link #ASSET_QUEUE}.+ * Send internal attribute change events into the {@link #ATTRIBUTE_EVENT_QUEUE}.*/public void sendAttributeEvent(AttributeEvent attributeEvent) {sendAttributeEvent(attributeEvent, INTERNAL);@@ -451,7 +440,7 @@ public class AssetProcessingService extends RouteBuilder implements ContainerSermessageBrokerService.getFluentProducerTemplate().withHeader(HEADER_SOURCE, source).withBody(attributeEvent)- .to(ASSET_QUEUE)+ .to(ATTRIBUTE_EVENT_QUEUE).asyncSend();}diff --git a/manager/src/main/java/org/openremote/manager/asset/AssetResourceImpl.java b/manager/src/main/java/org/openremote/manager/asset/AssetResourceImpl.javaindex f308fc658..375d7610d 100644--- a/manager/src/main/java/org/openremote/manager/asset/AssetResourceImpl.java+++ b/manager/src/main/java/org/openremote/manager/asset/AssetResourceImpl.java@@ -52,6 +52,8 @@ import java.util.logging.Logger;import java.util.stream.IntStream;import static jakarta.ws.rs.core.Response.Status.*;+import static org.openremote.manager.asset.AssetProcessingService.ATTRIBUTE_EVENT_QUEUE;+import static org.openremote.manager.event.ClientEventService.CLIENT_INBOUND_QUEUE;import static org.openremote.model.attribute.AttributeEvent.Source.CLIENT;import static org.openremote.model.query.AssetQuery.Access;import static org.openremote.model.value.MetaItemType.*;@@ -599,11 +601,10 @@ public class AssetResourceImpl extends ManagerWebResource implements AssetResourObject result = messageBrokerService.getFluentProducerTemplate().withBody(event).withHeader(AttributeEvent.HEADER_SOURCE, CLIENT)- .to(AssetProcessingService.ASSET_QUEUE)+ .to(ATTRIBUTE_EVENT_QUEUE).request();- if (result instanceof AssetProcessingException) {- AssetProcessingException processingException = (AssetProcessingException) result;+ if (result instanceof AssetProcessingException processingException) {failure = processingException.getReason();}diff --git a/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.java b/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.javaindex f21be9a35..26262f56a 100644--- a/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.java+++ b/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.java@@ -79,7 +79,7 @@ import static java.util.logging.Level.FINE;import static java.util.stream.Collectors.groupingBy;import static org.openremote.container.persistence.PersistenceService.PERSISTENCE_TOPIC;import static org.openremote.container.persistence.PersistenceService.isPersistenceEventForEntityType;-import static org.openremote.manager.event.ClientEventService.CLIENT_EVENT_TOPIC;+import static org.openremote.manager.event.ClientEventService.CLIENT_INBOUND_QUEUE;import static org.openremote.model.attribute.Attribute.getAddedOrModifiedAttributes;import static org.openremote.model.query.AssetQuery.*;import static org.openremote.model.query.AssetQuery.Access.*;@@ -378,13 +378,13 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicpublic void configure() throws Exception {// If any asset was modified in the database, publish eventsfrom(PERSISTENCE_TOPIC)- .routeId("AssetPersistenceChanges")+ .routeId("Persistence-Asset").filter(isPersistenceEventForEntityType(Asset.class)).process(exchange -> publishModificationEvents(exchange.getIn().getBody(PersistenceEvent.class)));// React if a client wants to read assets and attributes- from(CLIENT_EVENT_TOPIC)- .routeId("FromClientReadRequests")+ from(CLIENT_INBOUND_QUEUE)+ .routeId("ClientInbound-Query").filter(body().isInstanceOf(HasAssetQuery.class)).process(exchange -> {HasAssetQuery hasAssetQuery = exchange.getIn().getBody(HasAssetQuery.class);diff --git a/manager/src/main/java/org/openremote/manager/asset/AssetUpdateProcessor.java b/manager/src/main/java/org/openremote/manager/asset/AssetUpdateProcessor.javaindex 3c6b8b882..5ba3bc8c5 100644--- a/manager/src/main/java/org/openremote/manager/asset/AssetUpdateProcessor.java+++ b/manager/src/main/java/org/openremote/manager/asset/AssetUpdateProcessor.java@@ -19,7 +19,6 @@*/package org.openremote.manager.asset;-import org.openremote.container.concurrent.GlobalLock;import org.openremote.model.asset.Asset;import org.openremote.model.attribute.Attribute;import org.openremote.model.attribute.AttributeEvent.Source;@@ -28,11 +27,6 @@ import jakarta.persistence.EntityManager;/*** Process update of an asset attribute (value), from a {@link Source}.- *- * Implementations don't have to lock the whole context through {@link GlobalLock}, the- * caller of this has already obtained the lock. Internal synchronization and thread-safety- * must still be maintained if shared state of a processor can be modified by concurrent tasks- * maintained by that processor.*/public interface AssetUpdateProcessor {diff --git a/manager/src/main/java/org/openremote/manager/asset/ForecastService.java b/manager/src/main/java/org/openremote/manager/asset/ForecastService.javaindex 3f14aae3b..386d75475 100644--- a/manager/src/main/java/org/openremote/manager/asset/ForecastService.java+++ b/manager/src/main/java/org/openremote/manager/asset/ForecastService.java@@ -19,6 +19,7 @@*/package org.openremote.manager.asset;+import jakarta.persistence.TypedQuery;import org.apache.camel.builder.RouteBuilder;import org.openremote.container.message.MessageBrokerService;import org.openremote.container.persistence.PersistenceService;@@ -44,23 +45,13 @@ import org.openremote.model.value.ForecastConfiguration;import org.openremote.model.value.ForecastConfigurationWeightedExponentialAverage;import org.openremote.model.value.MetaItemType;-import jakarta.persistence.TypedQuery;import java.math.BigDecimal;import java.math.BigInteger;import java.time.Duration;import java.time.Instant;import java.time.LocalDateTime;import java.time.ZoneId;-import java.util.ArrayList;-import java.util.Collections;-import java.util.Date;-import java.util.HashMap;-import java.util.HashSet;-import java.util.List;-import java.util.Map;-import java.util.Optional;-import java.util.OptionalLong;-import java.util.Set;+import java.util.*;import java.util.concurrent.ScheduledExecutorService;import java.util.concurrent.ScheduledFuture;import java.util.concurrent.TimeUnit;@@ -70,7 +61,6 @@ import java.util.stream.Collectors;import java.util.stream.IntStream;import static java.util.stream.Collectors.toList;-import static org.openremote.container.concurrent.GlobalLock.withLockReturning;import static org.openremote.container.persistence.PersistenceService.PERSISTENCE_TOPIC;import static org.openremote.container.persistence.PersistenceService.isPersistenceEventForEntityType;import static org.openremote.manager.gateway.GatewayService.isNotForGateway;@@ -144,7 +134,7 @@ public class ForecastService extends RouteBuilder implements ContainerService {@Overridepublic void configure() throws Exception {from(PERSISTENCE_TOPIC)- .routeId("ForecastConfigurationPersistenceChanges")+ .routeId("Persistence-ForecastConfiguration").filter(isPersistenceEventForEntityType(Asset.class)).filter(isNotForGateway(gatewayService)).process(exchange -> {@@ -154,18 +144,16 @@ public class ForecastService extends RouteBuilder implements ContainerService {}protected List<Asset<?>> getForecastAssets() {- return withLockReturning(getClass().getSimpleName() + "::getForecastAssets", () -> {- return assetStorageService.findAll(- new AssetQuery().attributes(- new AttributePredicate().meta(- new NameValuePredicate(- FORECAST,- new StringPredicate(AssetQuery.Match.CONTAINS, true, "type")- )+ return assetStorageService.findAll(+ new AssetQuery().attributes(+ new AttributePredicate().meta(+ new NameValuePredicate(+ FORECAST,+ new StringPredicate(AssetQuery.Match.CONTAINS, true, "type")))- );- });+ )+ );}protected void processAssetChange(PersistenceEvent<Asset<?>> persistenceEvent) {diff --git a/manager/src/main/java/org/openremote/manager/asset/console/ConsoleResourceImpl.java b/manager/src/main/java/org/openremote/manager/asset/console/ConsoleResourceImpl.javaindex 1fb312a44..2ced44b16 100644--- a/manager/src/main/java/org/openremote/manager/asset/console/ConsoleResourceImpl.java+++ b/manager/src/main/java/org/openremote/manager/asset/console/ConsoleResourceImpl.java@@ -19,6 +19,7 @@*/package org.openremote.manager.asset.console;+import jakarta.ws.rs.BadRequestException;import org.openremote.container.timer.TimerService;import org.openremote.manager.asset.AssetStorageService;import org.openremote.manager.event.ClientEventService;@@ -29,7 +30,6 @@ import org.openremote.model.asset.AssetEvent;import org.openremote.model.asset.UserAssetLink;import org.openremote.model.asset.impl.ConsoleAsset;import org.openremote.model.asset.impl.GroupAsset;-import org.openremote.model.attribute.MetaItem;import org.openremote.model.console.ConsoleProviders;import org.openremote.model.console.ConsoleRegistration;import org.openremote.model.console.ConsoleResource;@@ -37,25 +37,20 @@ import org.openremote.model.http.RequestParams;import org.openremote.model.query.AssetQuery;import org.openremote.model.query.filter.AttributePredicate;import org.openremote.model.query.filter.ParentPredicate;-import org.openremote.model.query.filter.StringPredicate;import org.openremote.model.query.filter.RealmPredicate;+import org.openremote.model.query.filter.StringPredicate;import org.openremote.model.security.Realm;import org.openremote.model.util.TextUtil;-import org.openremote.model.util.ValueUtil;-import jakarta.ws.rs.BadRequestException;import java.util.Collections;-import java.util.HashMap;import java.util.Map;import java.util.Objects;--import static org.openremote.container.concurrent.GlobalLock.withLockReturning;-import static org.openremote.model.value.MetaItemType.*;+import java.util.concurrent.ConcurrentHashMap;public class ConsoleResourceImpl extends ManagerWebResource implements ConsoleResource {public static final String CONSOLE_PARENT_ASSET_NAME = "Consoles";- protected Map<String, String> realmConsoleParentMap = new HashMap<>();+ protected Map<String, String> realmConsoleParentMap = new ConcurrentHashMap<>();protected AssetStorageService assetStorageService;public ConsoleResourceImpl(TimerService timerService, ManagerIdentityService identityService, AssetStorageService assetStorageService, ClientEventService clientEventService) {@@ -148,17 +143,15 @@ public class ConsoleResourceImpl extends ManagerWebResource implements ConsoleRe}public String getConsoleParentAssetId(String realm) {- return withLockReturning(getClass().getSimpleName() + "::getConsoleParentAssetId", () -> {- String id = realmConsoleParentMap.get(realm);+ String id = realmConsoleParentMap.get(realm);- if (TextUtil.isNullOrEmpty(id)) {- Asset<?> consoleParent = getConsoleParentAsset(assetStorageService, getRequestRealm());- id = consoleParent.getId();- realmConsoleParentMap.put(realm, id);- }+ if (TextUtil.isNullOrEmpty(id)) {+ Asset<?> consoleParent = getConsoleParentAsset(assetStorageService, getRequestRealm());+ id = consoleParent.getId();+ realmConsoleParentMap.put(realm, id);+ }- return id;- });+ return id;}public static Asset<?> getConsoleParentAsset(AssetStorageService assetStorageService, Realm realm) {diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointHealthStatusProvider.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointHealthStatusProvider.javaindex 1ef7af997..46ee2536b 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointHealthStatusProvider.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointHealthStatusProvider.java@@ -25,7 +25,7 @@ import org.openremote.model.ContainerService;import org.openremote.model.system.HealthStatusProvider;import org.openremote.model.util.ValueUtil;-public class AssetDatapointHealthStatusProvider implements HealthStatusProvider, ContainerService {+public class AssetDatapointHealthStatusProvider implements HealthStatusProvider {public static final String NAME = "datapoints";protected AssetDatapointService assetDatapointService;diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointHealthStatusProvider.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointHealthStatusProvider.javaindex 1f2930231..9b244ca91 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointHealthStatusProvider.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointHealthStatusProvider.java@@ -25,7 +25,7 @@ import org.openremote.model.ContainerService;import org.openremote.model.system.HealthStatusProvider;import org.openremote.model.util.ValueUtil;-public class AssetPredictedDatapointHealthStatusProvider implements HealthStatusProvider, ContainerService {+public class AssetPredictedDatapointHealthStatusProvider implements HealthStatusProvider {public static final String NAME = "predicted";protected AssetPredictedDatapointService assetPredictedDatapointService;diff --git a/manager/src/main/java/org/openremote/manager/energy/EnergyOptimisationService.java b/manager/src/main/java/org/openremote/manager/energy/EnergyOptimisationService.javaindex b5b6f06b9..9add97824 100644--- a/manager/src/main/java/org/openremote/manager/energy/EnergyOptimisationService.java+++ b/manager/src/main/java/org/openremote/manager/energy/EnergyOptimisationService.java@@ -148,7 +148,7 @@ public class EnergyOptimisationService extends RouteBuilder implements Container@Overridepublic void configure() throws Exception {from(PERSISTENCE_TOPIC)- .routeId("EnergyOptimisationAssetPersistenceChanges")+ .routeId("Persistence-EnergyOptimisation").filter(isPersistenceEventForEntityType(EnergyOptimisationAsset.class)).filter(isNotForGateway(gatewayService)).process(exchange -> processAssetChange((PersistenceEvent<EnergyOptimisationAsset>) exchange.getIn().getBody(PersistenceEvent.class)));diff --git a/manager/src/main/java/org/openremote/manager/energy/ForecastSolarService.java b/manager/src/main/java/org/openremote/manager/energy/ForecastSolarService.javaindex e2641d054..4a143a15b 100644--- a/manager/src/main/java/org/openremote/manager/energy/ForecastSolarService.java+++ b/manager/src/main/java/org/openremote/manager/energy/ForecastSolarService.java@@ -94,7 +94,7 @@ public class ForecastSolarService extends RouteBuilder implements ContainerServi@Overridepublic void configure() throws Exception {from(PERSISTENCE_TOPIC)- .routeId("ForecastSolarAssetPersistenceChanges")+ .routeId("Persistence-ForecastSolar").filter(isPersistenceEventForEntityType(ElectricityProducerSolarAsset.class)).filter(isNotForGateway(gatewayService)).process(exchange -> processAssetChange((PersistenceEvent<ElectricityProducerSolarAsset>) exchange.getIn().getBody(PersistenceEvent.class)));diff --git a/manager/src/main/java/org/openremote/manager/energy/ForecastWindService.java b/manager/src/main/java/org/openremote/manager/energy/ForecastWindService.javaindex 054419e04..66437c234 100644--- a/manager/src/main/java/org/openremote/manager/energy/ForecastWindService.java+++ b/manager/src/main/java/org/openremote/manager/energy/ForecastWindService.java@@ -128,7 +128,7 @@ public class ForecastWindService extends RouteBuilder implements ContainerServic@Overridepublic void configure() throws Exception {from(PERSISTENCE_TOPIC)- .routeId("ForecastWindAssetPersistenceChanges")+ .routeId("Persistence-ForecastWind").filter(isPersistenceEventForEntityType(ElectricityProducerWindAsset.class)).filter(isNotForGateway(gatewayService)).process(exchange -> processAssetChange((PersistenceEvent<ElectricityProducerWindAsset>) exchange.getIn().getBody(PersistenceEvent.class)));diff --git a/manager/src/main/java/org/openremote/manager/event/ClientEventService.java b/manager/src/main/java/org/openremote/manager/event/ClientEventService.javaindex 1752a62c5..29ae7893e 100644--- a/manager/src/main/java/org/openremote/manager/event/ClientEventService.java+++ b/manager/src/main/java/org/openremote/manager/event/ClientEventService.java@@ -33,12 +33,12 @@ import org.openremote.container.security.basic.BasicAuthContext;import org.openremote.container.security.keycloak.AccessTokenAuthContext;import org.openremote.container.timer.TimerService;import org.openremote.manager.gateway.GatewayService;-import org.openremote.manager.mqtt.DefaultMQTTHandler;import org.openremote.manager.security.ManagerIdentityService;import org.openremote.manager.web.ManagerWebService;import org.openremote.model.Constants;import org.openremote.model.Container;import org.openremote.model.ContainerService;+import org.openremote.model.attribute.AttributeEvent;import org.openremote.model.event.shared.*;import org.openremote.model.syslog.SyslogEvent;@@ -54,10 +54,12 @@ import java.util.concurrent.CopyOnWriteArraySet;import java.util.concurrent.ExecutorService;import java.util.function.Consumer;-import static java.lang.System.Logger.Level.DEBUG;-import static java.lang.System.Logger.Level.TRACE;+import static java.lang.System.Logger.Level.*;import static org.apache.camel.builder.PredicateBuilder.or;+import static org.openremote.manager.asset.AssetProcessingService.ATTRIBUTE_EVENT_QUEUE;import static org.openremote.model.Constants.*;+import static org.openremote.model.attribute.AttributeEvent.HEADER_SOURCE;+import static org.openremote.model.attribute.AttributeEvent.Source.CLIENT;/*** Receives and publishes messages, handles the client/server event bus.@@ -101,7 +103,6 @@ import static org.openremote.model.Constants.*;* </p></dd>* </dl>*/-// TODO: Implement session expiry based on security principalpublic class ClientEventService extends RouteBuilder implements ContainerService {protected static class SessionInfo {@@ -119,16 +120,12 @@ public class ClientEventService extends RouteBuilder implements ContainerServicepublic static final String HEADER_CONNECTION_TYPE_WEBSOCKET = "websocket";public static final String HEADER_CONNECTION_TYPE_MQTT = "mqtt";public static final String HEADER_REQUEST_RESPONSE_MESSAGE_ID = ClientEventService.class.getName() + ".HEADER_REQUEST_RESPONSE_MESSAGE_ID";- private static final System.Logger LOG = System.getLogger(ClientEventService.class.getName());public static final String WEBSOCKET_URI = "undertow://ws://0.0.0.0/websocket/events?fireWebSocketChannelEvents=true&sendTimeout=15000"; // Host is not used as existing undertow instance is utilised+ public static final String CLIENT_INBOUND_QUEUE = "seda://ClientInboundQueue?multipleConsumers=true&concurrentConsumers=2&waitForTaskToComplete=IfReplyExpected&purgeWhenStopping=true&discardIfNoConsumers=true&limitConcurrentConsumers=false&size=1000";+ public static final String CLIENT_OUTBOUND_QUEUE = "seda://ClientOutboundQueue?multipleConsumers=true&concurrentConsumers=2&purgeWhenStopping=true&discardIfNoConsumers=true&limitConcurrentConsumers=false&size=1000";+ protected static final System.Logger LOG = System.getLogger(ClientEventService.class.getName());protected static final String INTERNAL_SESSION_KEY = "ClientEventServiceInternal";-- // TODO: Some of these options should be configurable depending on expected load etc.- public static final String CLIENT_EVENT_TOPIC = "seda://ClientEventTopic?multipleConsumers=true&concurrentConsumers=1&waitForTaskToComplete=NEVER&purgeWhenStopping=true&discardIfNoConsumers=true&limitConcurrentConsumers=false&size=1000";--// public static final String CLIENT_EVENT_QUEUE = "seda://ClientEventQueue?multipleConsumers=false&waitForTaskToComplete=NEVER&purgeWhenStopping=true&discardIfNoConsumers=true&size=25000";- // A direct endpoint with threading which then makes it async- public static final String CLIENT_EVENT_QUEUE = "direct://ClientEventQueue";+ protected static final String PUBLISH_QUEUE = "seda://ClientPublishQueue?multipleConsumers=false&purgeWhenStopping=true&discardIfNoConsumers=true&size=1000";final protected Collection<EventSubscriptionAuthorizer> eventSubscriptionAuthorizers = new CopyOnWriteArraySet<>();final protected Collection<EventAuthorizer> eventAuthorizers = new CopyOnWriteArraySet<>();@@ -150,10 +147,6 @@ public class ClientEventService extends RouteBuilder implements ContainerServiceexchange.setRouteStop(true);}- public static boolean isInbound(Exchange exchange) {- return org.apache.camel.builder.Builder.header(HEADER_CONNECTION_TYPE).isNotNull().matches(exchange);- }-public static String getSessionKey(Exchange exchange) {return exchange.getIn().getHeader(SESSION_KEY, String.class);}@@ -185,12 +178,6 @@ public class ClientEventService extends RouteBuilder implements ContainerServicecontainer.getService(TimerService.class));-// // TODO: Replace this custom websocket component with camel undertow component once it is migrated to jakarta packages-// WebsocketComponent websocketComponent = new DefaultWebsocketComponent(-// container-// );-// messageBrokerService.getContext().addComponent(WebsocketComponent.NAME, websocketComponent);-UndertowComponent undertowWebsocketComponent = new UndertowComponent(messageBrokerService.getContext()) {@Overrideprotected org.apache.camel.component.undertow.UndertowHost createUndertowHost(UndertowHostKey key) {@@ -218,12 +205,15 @@ public class ClientEventService extends RouteBuilder implements ContainerService// TODO: Remove prefix and just use event type then use a subscription wrapper to pass subscription ID around@Overridepublic void configure() throws Exception {++ // Route for deserializing incoming websocket messages and normalising them for the INBOUND_CLIENT_QUEUEfrom(WEBSOCKET_URI)- .routeId("FromWebsocket")+ .routeId("ClientInbound-Websocket").process(exchange -> {String connectionKey = exchange.getIn().getHeader(UndertowConstants.CONNECTION_KEY, String.class);exchange.getIn().setHeader(HEADER_CONNECTION_TYPE, HEADER_CONNECTION_TYPE_WEBSOCKET);exchange.getIn().setHeader(SESSION_KEY, connectionKey);+ exchange.getIn().setHeader(HEADER_SOURCE, CLIENT);}).choice().when(header(UndertowConstants.EVENT_TYPE))@@ -246,7 +236,7 @@ public class ClientEventService extends RouteBuilder implements ContainerService} else if (principal instanceof BasicAuthContext) {authContext = (BasicAuthContext) principal;} else if (principal != null) {- LOG.log(System.Logger.Level.INFO, "Unsupported user principal type: " + principal);+ LOG.log(INFO, "Unsupported user principal type: " + principal);}// Push auth and realm into channel for future use@@ -286,19 +276,8 @@ public class ClientEventService extends RouteBuilder implements ContainerService}}})- .to(CLIENT_EVENT_QUEUE)+ .to(CLIENT_INBOUND_QUEUE).stop()- .when(body().isInstanceOf(SharedEvent.class))- .process(exchange -> {- // Set timestamp if not set- if (exchange.getIn().getBody() instanceof SharedEvent) {- SharedEvent event = exchange.getIn().getBody(SharedEvent.class);- // If there is no timestamp in event, set to system time- if (event.getTimestamp() <= 0) {- event.setTimestamp(timerService.getCurrentTimeMillis());- }- }- }).endChoice().end().process(exchange -> {@@ -351,18 +330,22 @@ public class ClientEventService extends RouteBuilder implements ContainerService}}})- .to(CLIENT_EVENT_QUEUE)+ .to(CLIENT_INBOUND_QUEUE).end();- from(CLIENT_EVENT_QUEUE)- .routeId("ClientEvents")- .threads().executorService(executorService)+ from(CLIENT_INBOUND_QUEUE)+ .routeId("ClientInbound-EventProcessor")+ .process(this::passToInterceptors).choice()+ .when(body().isInstanceOf(AttributeEvent.class))+ .to(ATTRIBUTE_EVENT_QUEUE)+ .stop()+ .endChoice().when(header(SESSION_OPEN)).process(exchange -> {String sessionKey = getSessionKey(exchange);+ LOG.log(TRACE, "Adding session: " + sessionKey);sessionKeyInfoMap.put(sessionKey, createSessionInfo(sessionKey, exchange));- passToInterceptors(exchange);}).stop().when(or(@@ -374,17 +357,15 @@ public class ClientEventService extends RouteBuilder implements ContainerServiceLOG.log(TRACE, "Removing session: " + sessionKey);sessionKeyInfoMap.remove(sessionKey);eventSubscriptions.cancelAll(sessionKey);- passToInterceptors(exchange);}).stop().end()- .process(this::passToInterceptors).choice().when(body().isInstanceOf(EventSubscription.class)).process(exchange -> {String sessionKey = getSessionKey(exchange);EventSubscription<?> subscription = exchange.getIn().getBody(EventSubscription.class);- LOG.log(System.Logger.Level.TRACE, () -> "Adding subscription for session '" + sessionKey + "': " + subscription);+ LOG.log(TRACE, () -> "Adding subscription for session '" + sessionKey + "': " + subscription);eventSubscriptions.createOrUpdate(sessionKey, subscription);subscription.setSubscribed(true);sendToSession(sessionKey, subscription);@@ -395,25 +376,38 @@ public class ClientEventService extends RouteBuilder implements ContainerServiceString sessionKey = getSessionKey(exchange);CancelEventSubscription cancelEventSubscription = exchange.getIn().getBody(CancelEventSubscription.class);eventSubscriptions.cancel(sessionKey, cancelEventSubscription);- LOG.log(System.Logger.Level.TRACE, () -> "Cancelling subscription for session '" + sessionKey + "': " + cancelEventSubscription);+ LOG.log(TRACE, () -> "Cancelling subscription for session '" + sessionKey + "': " + cancelEventSubscription);}).stop()- .when(body().isInstanceOf(SharedEvent.class))- .choice()- .when(header(HEADER_CONNECTION_TYPE).isNotNull()) // Inbound messages from client- .to(ClientEventService.CLIENT_EVENT_TOPIC)- .stop()- .when(header(HEADER_CONNECTION_TYPE).isNull()) // Outbound message to clients+ .end();++ // Split publish messages for individual subscribers+ from(PUBLISH_QUEUE)+ .routeId("ClientOutbound-Splitter").split(method(eventSubscriptions, "splitForSubscribers")).process(exchange -> {String sessionKey = getSessionKey(exchange);- sendToSession(sessionKey, exchange.getIn().getBody());+ SessionInfo sessionInfo = sessionKeyInfoMap.get(sessionKey);+ if (sessionInfo == null) {+ LOG.log(INFO, "Cannot send to requested session it doesn't exist or is disconnected:" + sessionKey);+ return;+ }+ exchange.getIn().setHeader(HEADER_CONNECTION_TYPE, sessionInfo.connectionType);})- .stop()- .endChoice()- .otherwise()- .process(exchange -> LOG.log(System.Logger.Level.TRACE, () -> "Unsupported message body: " + exchange.getIn().getBody()))- .end();+ .to(CLIENT_OUTBOUND_QUEUE);++ // Route messages destined for websocket clients+ from(CLIENT_OUTBOUND_QUEUE)+ .routeId("ClientOutbound-Websocket")+ .filter(header(HEADER_CONNECTION_TYPE).isEqualTo(HEADER_CONNECTION_TYPE_WEBSOCKET))+ .process(exchange -> {+ String sessionKey = exchange.getIn().getHeader(SESSION_KEY, String.class);+ messageBrokerService.getFluentProducerTemplate()+ .withBody(exchange.getIn().getBody())+ .withHeader(UndertowConstants.CONNECTION_KEY, sessionKey)+ .to(WEBSOCKET_URI)+ .asyncSend();+ });}/**@@ -490,23 +484,28 @@ public class ClientEventService extends RouteBuilder implements ContainerService}/**- * This handles basic authorisation checks for clients that want to write an event to the system+ * This handles basic authorisation checks for clients that want to write an event to the system; this gets hit a lot+ * so should be as performant as possible*/+ // TODO: Implement auth cache in OR that covers HTTP, WS and MQTT (ActiveMQ Authorization cache currently covers MQTT which is the main use case)public <T extends SharedEvent> boolean authorizeEventWrite(String realm, AuthContext authContext, T event) {boolean authorized = eventAuthorizers.stream().anyMatch(authorizer -> authorizer.authorise(realm, authContext, event));if (!authorized) {if (authContext != null) {- LOG.log(DEBUG, "Client not authorised to send event: type=" + event.getEventType() + ", requestRealm=" + realm + ", username=" + authContext.getUsername() + ", userRealm=" + authContext.getAuthenticatedRealmName());+ LOG.log(DEBUG, () -> "Client not authorised to send event: type=" + event.getEventType() + ", requestRealm=" + realm + ", user=" + authContext.getUsername() + ", userRealm=" + authContext.getAuthenticatedRealmName());} else {- LOG.log(DEBUG,"Client not authorised to send event: type=" + event.getEventType() + ", requestRealm=" + realm + ", user=null");+ LOG.log(DEBUG, () -> "Client not authorised to send event: type=" + event.getEventType() + ", requestRealm=" + realm + ", user=null");}}return authorized;}+ /**+ * Publish an event to interested clients+ */public <T extends SharedEvent> void publishEvent(T event) {// Only publish if service is not stoppedif (stopped) {@@ -520,31 +519,25 @@ public class ClientEventService extends RouteBuilder implements ContainerService}messageBrokerService.getFluentProducerTemplate().withBody(event)- .to(CLIENT_EVENT_QUEUE)+ .to(PUBLISH_QUEUE).asyncSend();}}public void sendToSession(String sessionKey, Object data) {if (messageBrokerService != null && messageBrokerService.getFluentProducerTemplate() != null) {+ LOG.log(TRACE, () -> "Sending to session '" + sessionKey + "': " + data);SessionInfo sessionInfo = sessionKeyInfoMap.get(sessionKey);if (sessionInfo == null) {- LOG.log(System.Logger.Level.INFO, "Cannot send to requested session it doesn't exist or is disconnected:" + sessionKey);+ LOG.log(INFO, "Cannot send to requested session it doesn't exist or is disconnected:" + sessionKey);return;}- if (sessionInfo.connectionType.equals(HEADER_CONNECTION_TYPE_WEBSOCKET)) {- messageBrokerService.getFluentProducerTemplate()- .withBody(data)- .withHeader(UndertowConstants.CONNECTION_KEY, sessionKey)- .to(WEBSOCKET_URI)- .asyncSend();- } else if (sessionInfo.connectionType.equals(HEADER_CONNECTION_TYPE_MQTT)) {- messageBrokerService.getFluentProducerTemplate()- .withBody(data)- .withHeader(SESSION_KEY, sessionKey)- .to(DefaultMQTTHandler.CLIENT_QUEUE)- .asyncSend();- }+ messageBrokerService.getFluentProducerTemplate()+ .withBody(data)+ .withHeader(SESSION_KEY, sessionKey)+ .withHeader(HEADER_CONNECTION_TYPE, sessionInfo.connectionType)+ .to(CLIENT_OUTBOUND_QUEUE)+ .asyncSend();}}@@ -559,16 +552,17 @@ public class ClientEventService extends RouteBuilder implements ContainerServicesessionInfo.closeRunnable.run();}+ @SuppressWarnings("ResultOfMethodCallIgnored")protected void passToInterceptors(Exchange exchange) {// Pass to each interceptor and stop if any interceptor marks the exchange as stop routing- if (exchangeInterceptors.stream().anyMatch(interceptor -> {+ exchangeInterceptors.stream().anyMatch(interceptor -> {interceptor.accept(exchange);boolean stop = exchange.isRouteStop();if (stop) {LOG.log(TRACE, "Client event interceptor marked exchange as `stop routing`: " + interceptor);}return stop;- }));+ });}@Override@@ -583,6 +577,9 @@ public class ClientEventService extends RouteBuilder implements ContainerServicereturn new SessionInfo(connectionType, closeRunnable);}+ /**+ * Provides a protocol agnostic way to close a client session+ */protected Runnable getWebsocketSessionTerminator(WebSocketChannel webSocketChannel) {return () -> {try {diff --git a/manager/src/main/java/org/openremote/manager/event/EventSubscriptions.java b/manager/src/main/java/org/openremote/manager/event/EventSubscriptions.javaindex c817b7c7c..f58a6c09e 100644--- a/manager/src/main/java/org/openremote/manager/event/EventSubscriptions.java+++ b/manager/src/main/java/org/openremote/manager/event/EventSubscriptions.java@@ -104,6 +104,11 @@ public class EventSubscriptions {}LOG.finest("Cancel subscription for session '" + sessionKey + "': " + subscription);SessionSubscriptions sessionSubscriptions = this.sessionSubscriptionIdMap.get(sessionKey);++ if (sessionSubscriptions == null) {+ return;+ }+if (!TextUtil.isNullOrEmpty(subscription.getSubscriptionId())) {sessionSubscriptions.cancelById(subscription.getSubscriptionId());} else {@@ -121,6 +126,9 @@ public class EventSubscriptions {}}+ /**+ * Used in {@link ClientEventService#PUBLISH_QUEUE}+ */@SuppressWarnings({"unchecked", "unused"})public <T extends SharedEvent> List<Message> splitForSubscribers(Exchange exchange) {List<Message> messageList = new ArrayList<>();diff --git a/manager/src/main/java/org/openremote/manager/gateway/GatewayClientService.java b/manager/src/main/java/org/openremote/manager/gateway/GatewayClientService.javaindex 4587193fa..ce3782b4d 100644--- a/manager/src/main/java/org/openremote/manager/gateway/GatewayClientService.java+++ b/manager/src/main/java/org/openremote/manager/gateway/GatewayClientService.java@@ -148,7 +148,7 @@ public class GatewayClientService extends RouteBuilder implements ContainerServipublic void configure() throws Exception {from(PERSISTENCE_TOPIC)- .routeId("GatewayServiceConnectionChanges")+ .routeId("Persistence-GatewayConnection").filter(isPersistenceEventForEntityType(GatewayConnection.class)).process(exchange -> {@SuppressWarnings("unchecked")diff --git a/manager/src/main/java/org/openremote/manager/gateway/GatewayService.java b/manager/src/main/java/org/openremote/manager/gateway/GatewayService.javaindex d1a02c2e7..e12e95980 100644--- a/manager/src/main/java/org/openremote/manager/gateway/GatewayService.java+++ b/manager/src/main/java/org/openremote/manager/gateway/GatewayService.java@@ -245,7 +245,7 @@ public class GatewayService extends RouteBuilder implements ContainerService, Asif (active) {from(PERSISTENCE_TOPIC)- .routeId("GatewayServiceAssetChanges")+ .routeId("Persistence-GatewayAsset").filter(isPersistenceEventForEntityType(Asset.class)).process(exchange -> {@SuppressWarnings("unchecked")@@ -307,7 +307,7 @@ public class GatewayService extends RouteBuilder implements ContainerService, As}// Inbound shared events- if (and(ClientEventService::isInbound, body().isInstanceOf(SharedEvent.class)).matches(exchange)) {+ if (body().isInstanceOf(SharedEvent.class).matches(exchange)) {ClientEventService.stopMessage(exchange);String gatewayId = getGatewayIdFromClientId(clientId);onGatewayClientEventReceived(gatewayId, exchange.getIn().getHeader(ClientEventService.HEADER_REQUEST_RESPONSE_MESSAGE_ID, String.class), exchange.getIn().getBody(SharedEvent.class));diff --git a/manager/src/main/java/org/openremote/manager/mqtt/ActiveMQORSecurityManager.java b/manager/src/main/java/org/openremote/manager/mqtt/ActiveMQORSecurityManager.javaindex 3b8b78e2c..b32e4a0a0 100644--- a/manager/src/main/java/org/openremote/manager/mqtt/ActiveMQORSecurityManager.java+++ b/manager/src/main/java/org/openremote/manager/mqtt/ActiveMQORSecurityManager.java@@ -158,7 +158,7 @@ public class ActiveMQORSecurityManager extends ActiveMQJAASSecurityManager {// Get MQTT topic from addresstopic = Topic.fromAddress(address, brokerService.getWildcardConfiguration());} catch (IllegalArgumentException e) {- LOG.log(Level.INFO, "Invalid topic provided by client '" + address, e);+ LOG.log(Level.FINE, "Invalid topic provided by client '" + address, e);return false;}@@ -166,7 +166,7 @@ public class ActiveMQORSecurityManager extends ActiveMQJAASSecurityManager {String topicClientID = MQTTHandler.topicClientID(topic);if (topicClientID == null) {- LOG.info("Client ID not found but it must be included as the second token in the topic: topic=" + topic);+ LOG.fine("Client ID not found but it must be included as the second token in the topic: topic=" + topic);return false;}diff --git a/manager/src/main/java/org/openremote/manager/mqtt/ConnectionMonitorHandler.java b/manager/src/main/java/org/openremote/manager/mqtt/ConnectionMonitorHandler.javaindex a8457fbe8..7ce059a29 100644--- a/manager/src/main/java/org/openremote/manager/mqtt/ConnectionMonitorHandler.java+++ b/manager/src/main/java/org/openremote/manager/mqtt/ConnectionMonitorHandler.java@@ -93,7 +93,7 @@ public class ConnectionMonitorHandler extends MQTTHandler {@Overridepublic void configure() throws Exception {from(PERSISTENCE_TOPIC)- .routeId("MQTTConnectionMonitorHandlerQueue")+ .routeId("Persistence-MQTTConnectedAttributes").filter(isPersistenceEventForEntityType(Asset.class)).filter(isNotForGateway(gatewayService)).process(exchange -> {@@ -248,7 +248,7 @@ public class ConnectionMonitorHandler extends MQTTHandler {}protected void addSessionAttributes(String realm, List<Pair<String, Attribute<?>>> assetIdsAttrs) {- LOG.fine("Adding '" + assetIdsAttrs.size() + "' attributes(s) with user linked attributes in realm: " + realm);+ LOG.finest("Adding '" + assetIdsAttrs.size() + "' attributes(s) with user linked attributes in realm: " + realm);List<String> usernames = assetIdsAttrs.stream().map(assetIdAttr -> assetIdAttr.getValue().getMetaValue(USER_CONNECTED).orElse(null)).filter(Objects::nonNull)diff --git a/manager/src/main/java/org/openremote/manager/mqtt/DefaultMQTTHandler.java b/manager/src/main/java/org/openremote/manager/mqtt/DefaultMQTTHandler.javaindex 4da9b434c..e45a3d0b5 100644--- a/manager/src/main/java/org/openremote/manager/mqtt/DefaultMQTTHandler.java+++ b/manager/src/main/java/org/openremote/manager/mqtt/DefaultMQTTHandler.java@@ -19,18 +19,22 @@*/package org.openremote.manager.mqtt;+import com.google.common.cache.Cache;+import com.google.common.cache.CacheBuilder;import io.netty.buffer.ByteBuf;import io.netty.handler.codec.mqtt.MqttQoS;+import org.apache.activemq.artemis.api.core.SimpleString;import org.apache.activemq.artemis.spi.core.protocol.RemotingConnection;+import org.apache.activemq.artemis.utils.collections.ConcurrentHashSet;import org.apache.camel.builder.RouteBuilder;import org.keycloak.KeycloakSecurityContext;import org.openremote.container.security.AuthContext;-import org.openremote.manager.asset.AssetStorageService;import org.openremote.manager.event.ClientEventService;-import org.openremote.model.Constants;import org.openremote.model.Container;+import org.openremote.model.PersistenceEvent;import org.openremote.model.asset.AssetEvent;import org.openremote.model.asset.AssetFilter;+import org.openremote.model.asset.UserAssetLink;import org.openremote.model.attribute.AttributeEvent;import org.openremote.model.event.TriggeredEventSubscription;import org.openremote.model.event.shared.CancelEventSubscription;@@ -43,14 +47,18 @@ import java.nio.charset.StandardCharsets;import java.util.*;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ConcurrentMap;+import java.util.concurrent.TimeUnit;import java.util.function.Consumer;import java.util.function.Function;import java.util.logging.Logger;import java.util.regex.Pattern;-import static org.openremote.manager.event.ClientEventService.CLIENT_EVENT_QUEUE;+import static org.apache.camel.support.builder.PredicateBuilder.and;+import static org.openremote.manager.event.ClientEventService.*;import static org.openremote.manager.mqtt.MQTTBrokerService.getConnectionIDString;import static org.openremote.model.Constants.*;+import static org.openremote.model.attribute.AttributeEvent.HEADER_SOURCE;+import static org.openremote.model.attribute.AttributeEvent.Source.CLIENT;import static org.openremote.model.syslog.SyslogCategory.API;/**@@ -77,15 +85,18 @@ public class DefaultMQTTHandler extends MQTTHandler {}}- public static final String CLIENT_QUEUE = "seda://MQTTDefaultHandlerQueue?waitForTaskToComplete=IfReplyExpected&timeout=10000&purgeWhenStopping=true&discardIfNoConsumers=false&size=25000";public static final int PRIORITY = Integer.MIN_VALUE + 1000;public static final String ASSET_TOPIC = "asset";public static final String ATTRIBUTE_TOPIC = "attribute";public static final String ATTRIBUTE_VALUE_TOPIC = "attributevalue";public static final String ATTRIBUTE_VALUE_WRITE_TOPIC = "writeattributevalue";private static final Logger LOG = SyslogCategory.getLogger(API, DefaultMQTTHandler.class);- protected AssetStorageService assetStorageService;protected final ConcurrentMap<String, SubscriberInfo> connectionSubscriberInfoMap = new ConcurrentHashMap<>();+ // An authorisation cache for publishing+ protected final Cache<String, ConcurrentHashSet<String>> authorizationCache = CacheBuilder.newBuilder()+ .maximumSize(100000)+ .expireAfterWrite(300000, TimeUnit.MILLISECONDS)+ .build();@Overridepublic int getPriority() {@@ -99,40 +110,33 @@ public class DefaultMQTTHandler extends MQTTHandler {messageBrokerService.getContext().addRoutes(new RouteBuilder() {@Overridepublic void configure() throws Exception {- from(CLIENT_QUEUE)- .routeId("MQTTDefaultHandlerQueue")- .choice()- .when(body().isInstanceOf(TriggeredEventSubscription.class))++ // Route messages destined for MQTT clients+ from(CLIENT_OUTBOUND_QUEUE)+ .routeId("ClientOutbound-DefaultMQTTHandler")+ .filter(and(+ header(HEADER_CONNECTION_TYPE).isEqualTo(HEADER_CONNECTION_TYPE_MQTT),+ body().isInstanceOf(TriggeredEventSubscription.class)+ )).process(exchange -> {// Get the subscriber consumerString connectionID = exchange.getIn().getHeader(SESSION_KEY, String.class);- if (connectionID == null) {- LOG.warning("No connection ID for triggered subscription so cannot deliver");- } else {- SubscriberInfo subscriberInfo = connectionSubscriberInfoMap.get(connectionID);- if (subscriberInfo != null) {- TriggeredEventSubscription<?> triggeredEventSubscription = exchange.getIn().getBody(TriggeredEventSubscription.class);- String topic = triggeredEventSubscription.getSubscriptionId();- // Should only be a single event in here- SharedEvent event = triggeredEventSubscription.getEvents().get(0);- Consumer<SharedEvent> eventConsumer = subscriberInfo.topicSubscriptionMap.get(topic);- if (eventConsumer != null) {- eventConsumer.accept(event);- }+ SubscriberInfo subscriberInfo = connectionSubscriberInfoMap.get(connectionID);+ if (subscriberInfo != null) {+ TriggeredEventSubscription<?> triggeredEventSubscription = exchange.getIn().getBody(TriggeredEventSubscription.class);+ String topic = triggeredEventSubscription.getSubscriptionId();+ // Should only be a single event in here+ SharedEvent event = triggeredEventSubscription.getEvents().get(0);+ Consumer<SharedEvent> eventConsumer = subscriberInfo.topicSubscriptionMap.get(topic);+ if (eventConsumer != null) {+ eventConsumer.accept(event);}}- })- .end();+ });}});}- @Override- public void start(Container container) throws Exception {- super.start(container);- assetStorageService = container.getService(AssetStorageService.class);- }-@Overridepublic void onConnect(RemotingConnection connection) {super.onConnect(connection);@@ -149,7 +153,7 @@ public class DefaultMQTTHandler extends MQTTHandler {headers.put(SESSION_TERMINATOR, closeRunnable);messageBrokerService.getFluentProducerTemplate().withHeaders(headers)- .to(CLIENT_EVENT_QUEUE)+ .to(CLIENT_INBOUND_QUEUE).asyncSend();}@@ -161,9 +165,10 @@ public class DefaultMQTTHandler extends MQTTHandler {headers.put(SESSION_CLOSE, true);messageBrokerService.getFluentProducerTemplate().withHeaders(headers)- .to(CLIENT_EVENT_QUEUE)+ .to(CLIENT_INBOUND_QUEUE).asyncSend();connectionSubscriberInfoMap.remove(getConnectionIDString(connection));+ authorizationCache.invalidate(getConnectionIDString(connection));}@Override@@ -173,7 +178,7 @@ public class DefaultMQTTHandler extends MQTTHandler {headers.put(SESSION_CLOSE_ERROR, true);messageBrokerService.getFluentProducerTemplate().withHeaders(headers)- .to(CLIENT_EVENT_QUEUE)+ .to(CLIENT_INBOUND_QUEUE).asyncSend();connectionSubscriberInfoMap.remove(getConnectionIDString(connection));}@@ -221,45 +226,45 @@ public class DefaultMQTTHandler extends MQTTHandler {if (!Pattern.matches(ASSET_ID_REGEXP, topicTokenIndexToString(topic, 3))&& !TOKEN_MULTI_LEVEL_WILDCARD.equals(topicTokenIndexToString(topic, 3))&& !TOKEN_SINGLE_LEVEL_WILDCARD.equals(topicTokenIndexToString(topic, 3))) {- LOG.finest("Asset subscribe forth token must be an asset ID or wildcard: topic=" + topic + ", " + mqttBrokerService.connectionToString(connection));+ LOG.fine("Asset subscribe forth token must be an asset ID or wildcard: topic=" + topic + ", " + mqttBrokerService.connectionToString(connection));return false;}} else if (topic.getTokens().size() == 5) {if (!Pattern.matches(ASSET_ID_REGEXP, topicTokenIndexToString(topic, 3))) {- LOG.finest("Asset subscribe forth token must be an asset ID: topic=" + topic + ", " + mqttBrokerService.connectionToString(connection));+ LOG.fine("Asset subscribe forth token must be an asset ID: topic=" + topic + ", " + mqttBrokerService.connectionToString(connection));return false;}if (!TOKEN_MULTI_LEVEL_WILDCARD.equals(topicTokenIndexToString(topic, 4))&& !TOKEN_SINGLE_LEVEL_WILDCARD.equals(topicTokenIndexToString(topic, 4))) {- LOG.finest("Asset subscribe fifth token must be a wildcard: topic=" + topic + ", " + mqttBrokerService.connectionToString(connection));+ LOG.fine("Asset subscribe fifth token must be a wildcard: topic=" + topic + ", " + mqttBrokerService.connectionToString(connection));return false;}}} else {// Attribute topicif (topic.getTokens().size() < 5 || topic.getTokens().size() > 6) {- LOG.finest("Attribute subscribe token count should be 5 or 6: topic=" + topic + ", " + mqttBrokerService.connectionToString(connection));+ LOG.fine("Attribute subscribe token count should be 5 or 6: topic=" + topic + ", " + mqttBrokerService.connectionToString(connection));return false;}if (topic.getTokens().size() == 5) {if (TOKEN_MULTI_LEVEL_WILDCARD.equals(topicTokenIndexToString(topic, 3))) {- LOG.finest("Attribute subscribe multi level wildcard must be last token: topic=" + topic + ", " + mqttBrokerService.connectionToString(connection));+ LOG.fine("Attribute subscribe multi level wildcard must be last token: topic=" + topic + ", " + mqttBrokerService.connectionToString(connection));return false;}if (!Pattern.matches(ASSET_ID_REGEXP, topicTokenIndexToString(topic, 4))&& !TOKEN_MULTI_LEVEL_WILDCARD.equals(topicTokenIndexToString(topic, 4))&& !TOKEN_SINGLE_LEVEL_WILDCARD.equals(topicTokenIndexToString(topic, 4))) {- LOG.finest("Attribute subscribe fifth token must be an asset ID or a wildcard: topic=" + topic + ", " + mqttBrokerService.connectionToString(connection));+ LOG.fine("Attribute subscribe fifth token must be an asset ID or a wildcard: topic=" + topic + ", " + mqttBrokerService.connectionToString(connection));return false;}} else if (topic.getTokens().size() == 6) {if (!Pattern.matches(ASSET_ID_REGEXP, topicTokenIndexToString(topic, 4))) {- LOG.finest("Attribute subscribe fifth token must be an asset ID: topic=" + topic + ", " + mqttBrokerService.connectionToString(connection));+ LOG.fine("Attribute subscribe fifth token must be an asset ID: topic=" + topic + ", " + mqttBrokerService.connectionToString(connection));return false;}if (!TOKEN_MULTI_LEVEL_WILDCARD.equals(topicTokenIndexToString(topic, 5))&& !TOKEN_SINGLE_LEVEL_WILDCARD.equals(topicTokenIndexToString(topic, 5))) {- LOG.finest("Attribute subscribe sixth token must be a wildcard: topic=" + topic + ", " + mqttBrokerService.connectionToString(connection));+ LOG.fine("Attribute subscribe sixth token must be a wildcard: topic=" + topic + ", " + mqttBrokerService.connectionToString(connection));return false;}}@@ -286,37 +291,60 @@ public class DefaultMQTTHandler extends MQTTHandler {return true;}+ // TODO: improve authorisation performance+ // We make heavy use of authorisation caching as clients can hit this a lot and it is currently quite slow with DB calls@Overridepublic boolean canPublish(RemotingConnection connection, KeycloakSecurityContext securityContext, Topic topic) {if (!isKeycloak) {- LOG.finest("Identity provider is not keycloak");+ LOG.fine("Identity provider is not keycloak");return false;}AuthContext authContext = getAuthContextFromSecurityContext(securityContext);if (authContext == null) {- LOG.finest("Anonymous publish not supported: topic=" + topic + ", connection=" + mqttBrokerService.connectionToString(connection));+ LOG.finer("Anonymous publish not supported: topic=" + topic + ", connection=" + mqttBrokerService.connectionToString(connection));return false;}if (isAttributeValueWriteTopic(topic)) {if (topic.getTokens().size() != 5 || !Pattern.matches(ASSET_ID_REGEXP, topicTokenIndexToString(topic, 4))) {- LOG.finest("Publish attribute value topic should be {realm}/{clientId}/writeattributevalue/{attributeName}/{assetId}: topic=" + topic + ", connection=" + mqttBrokerService.connectionToString(connection));+ LOG.finer("Publish attribute value topic should be {realm}/{clientId}/writeattributevalue/{attributeName}/{assetId}: topic=" + topic + ", connection=" + mqttBrokerService.connectionToString(connection));return false;}} else {return false;}+ String cacheKey = getConnectionIDString(connection);++ // Check cache+ ConcurrentHashSet<String> act = authorizationCache.getIfPresent(cacheKey);+ if (act != null && act.contains(topic.getString())) {+ return true;+ }+// We don't know the value at this point so just use a null value for authorization (value type will be handled// when the event is processed)if (!clientEventService.authorizeEventWrite(topicRealm(topic), authContext, buildAttributeEvent(topic.getTokens(), null))) {- LOG.finest("Publish was not authorised for this user and topic: topic=" + topic + ", subject=" + authContext);+ LOG.fine("Publish was not authorised for this user and topic: topic=" + topic + ", subject=" + authContext);return false;}+ // Add to cache+ ConcurrentHashSet<String> set;+ synchronized (authorizationCache) {+ act = authorizationCache.getIfPresent(cacheKey);+ if (act != null) {+ set = act;+ } else {+ set = new ConcurrentHashSet<>();+ authorizationCache.put(cacheKey, set);+ }+ }+ set.add(topic.getString());+return true;}@@ -346,7 +374,7 @@ public class DefaultMQTTHandler extends MQTTHandler {messageBrokerService.getFluentProducerTemplate().withHeaders(headers).withBody(subscription)- .to(CLIENT_EVENT_QUEUE)+ .to(CLIENT_INBOUND_QUEUE).asyncSend();// Track connection subscriptions for restricted user asset link changes (to determine if the client should be disconnected)@@ -373,7 +401,7 @@ public class DefaultMQTTHandler extends MQTTHandler {messageBrokerService.getFluentProducerTemplate().withHeaders(headers).withBody(cancelEventSubscription)- .to(CLIENT_EVENT_QUEUE)+ .to(CLIENT_INBOUND_QUEUE).asyncSend();// Track connection subscriptions for restricted user asset link changes (to determine if the client should be disconnected)@@ -401,16 +429,21 @@ public class DefaultMQTTHandler extends MQTTHandler {Object value = ValueUtil.parse(payloadContent).orElse(null);AttributeEvent attributeEvent = buildAttributeEvent(topicTokens, value);Map<String, Object> headers = prepareHeaders(topicRealm(topic), connection);+ LOG.finer("Publishing to client inbound queue: " + attributeEvent);messageBrokerService.getFluentProducerTemplate().withHeaders(headers).withBody(attributeEvent)- .to(CLIENT_EVENT_QUEUE)+ .to(CLIENT_INBOUND_QUEUE).asyncSend();}@Override- public void onUserAssetLinksChanged(RemotingConnection connection) {+ public void onUserAssetLinksChanged(RemotingConnection connection, List<PersistenceEvent<UserAssetLink>> changes) {if (connectionSubscriberInfoMap.containsKey(getConnectionIDString(connection))) {+ if (changes.stream().allMatch(pe -> pe.getCause() == PersistenceEvent.Cause.CREATE)) {+ // Do nothing if only links have been added+ return;+ }LOG.info("User asset links have changed for a connected user with active subscriptions so force disconnecting them: " + mqttBrokerService.connectionToString(connection));mqttBrokerService.doForceDisconnect(connection);}@@ -572,8 +605,9 @@ public class DefaultMQTTHandler extends MQTTHandler {protected static Map<String, Object> prepareHeaders(String requestRealm, RemotingConnection connection) {Map<String, Object> headers = new HashMap<>();headers.put(SESSION_KEY, getConnectionIDString(connection));- headers.put(ClientEventService.HEADER_CONNECTION_TYPE, ClientEventService.HEADER_CONNECTION_TYPE_MQTT);- headers.put(Constants.REALM_PARAM_NAME, requestRealm);+ headers.put(HEADER_CONNECTION_TYPE, ClientEventService.HEADER_CONNECTION_TYPE_MQTT);+ headers.put(REALM_PARAM_NAME, requestRealm);+ headers.put(HEADER_SOURCE, CLIENT);return headers;}}diff --git a/manager/src/main/java/org/openremote/manager/mqtt/MQTTBrokerService.java b/manager/src/main/java/org/openremote/manager/mqtt/MQTTBrokerService.javaindex a47e16976..551d58ef1 100644--- a/manager/src/main/java/org/openremote/manager/mqtt/MQTTBrokerService.java+++ b/manager/src/main/java/org/openremote/manager/mqtt/MQTTBrokerService.java@@ -25,6 +25,7 @@ import io.netty.channel.ChannelId;import io.netty.handler.codec.mqtt.MqttQoS;import org.apache.activemq.artemis.api.core.ActiveMQException;import org.apache.activemq.artemis.api.core.ActiveMQExceptionType;+import org.apache.activemq.artemis.api.core.SimpleString;import org.apache.activemq.artemis.api.core.client.*;import org.apache.activemq.artemis.api.core.management.CoreNotificationType;import org.apache.activemq.artemis.api.core.management.ManagementHelper;@@ -36,10 +37,12 @@ import org.apache.activemq.artemis.core.config.impl.SecurityConfiguration;import org.apache.activemq.artemis.core.protocol.mqtt.MQTTUtil;import org.apache.activemq.artemis.core.remoting.FailureListener;import org.apache.activemq.artemis.core.remoting.impl.invm.InVMConnection;+import org.apache.activemq.artemis.core.security.impl.SecurityStoreImpl;import org.apache.activemq.artemis.core.server.ServerSession;import org.apache.activemq.artemis.core.server.embedded.EmbeddedActiveMQ;import org.apache.activemq.artemis.core.server.plugin.ActiveMQServerConnectionPlugin;import org.apache.activemq.artemis.core.server.plugin.ActiveMQServerSessionPlugin;+import org.apache.activemq.artemis.core.settings.impl.AddressSettings;import org.apache.activemq.artemis.spi.core.protocol.RemotingConnection;import org.apache.activemq.artemis.spi.core.security.jaas.GuestLoginModule;import org.apache.activemq.artemis.spi.core.security.jaas.PrincipalConversionLoginModule;@@ -111,6 +114,7 @@ public class MQTTBrokerService extends RouteBuilder implements ContainerService,protected List<MQTTHandler> customHandlers = new ArrayList<>();protected ConcurrentMap<String, RemotingConnection> clientIDConnectionMap = new ConcurrentHashMap<>();protected ConcurrentMap<String, RemotingConnection> connectionIDConnectionMap = new ConcurrentHashMap<>();+ protected ConcurrentMap<String, List<PersistenceEvent<UserAssetLink>>> userAssetLinkChangeMap = new ConcurrentHashMap<>();protected Debouncer<String> userAssetDisconnectDebouncer;// Stores disconnected connections for a short period to allow last will publishes to be processedprotected Cache<String, RemotingConnection> disconnectedConnectionCache;@@ -141,7 +145,7 @@ public class MQTTBrokerService extends RouteBuilder implements ContainerService,timerService = container.getService(TimerService.class);assetProcessingService = container.getService(AssetProcessingService.class);- userAssetDisconnectDebouncer = new Debouncer<>(executorService, this::processUserAssetLinkChange, debounceMillis);+ userAssetDisconnectDebouncer = new Debouncer<>(executorService, id -> processUserAssetLinkChange(id, userAssetLinkChangeMap.remove(id)), debounceMillis);disconnectedConnectionCache = CacheBuilder.newBuilder().maximumSize(10000).expireAfterWrite(10000, TimeUnit.MILLISECONDS)@@ -182,21 +186,36 @@ public class MQTTBrokerService extends RouteBuilder implements ContainerService,// Configure and start the brokerConfiguration config = new ConfigurationImpl();config.addAcceptorConfiguration("in-vm", "vm://0?protocols=core");- String serverURI = new URIBuilder().setScheme("tcp").setHost(host).setPort(port).setParameter("protocols", "MQTT").build().toString();+ String serverURI = new URIBuilder().setScheme("tcp").setHost(host).setPort(port)+ .setParameter("protocols", "MQTT")+ .setParameter("allowLinkStealing", "true")+ .setParameter("defaultMqttSessionExpiryInterval", "0") // Don't support retained sessions+ .build().toString();config.addAcceptorConfiguration("tcp", serverURI);- // TODO: Make this configurable- config.setMaxDiskUsage(-1);- config.setSecurityInvalidationInterval(30000);config.registerBrokerPlugin(this);-- // Register notification pluginconfig.setWildCardConfiguration(wildcardConfiguration);++ // Force all addresses to aggressively cleanup queues (don't support resumable sessions)+ config.addAddressSetting(wildcardConfiguration.getAnyWordsString(),+ new AddressSettings()+ .setDeadLetterAddress(SimpleString.toSimpleString("ActiveMQ.DLQ"))+ .setExpiryAddress(SimpleString.toSimpleString("ActiveMQ.expired"))+ .setAutoDeleteCreatedQueues(true)+ .setAutoDeleteAddresses(true)+ // Auto delete MQTT addresses after 1 day as they never get flagged as used so will linger otherwise+ .setAutoDeleteAddressesSkipUsageCheck(true)+ .setAutoDeleteAddressesDelay(86400000)+ .setAutoDeleteQueuesDelay(0)+ .setAutoDeleteQueuesMessageCount(-1L)+ );+config.setPersistenceEnabled(false);// TODO: Make auto provisioning clients disconnect and reconnect with credentials or pass through X.509 certificates for auth- // Disable auth cache so we can inject RemotingConnection into the subject as a principal otherwise we don't have access to the connection during authorisation so we cannot check topic contains the clientID- // Disable caching so we can support auto provisioning sessions without having to reconnect - we need to re-evaluate this and clients should probably send X.509 through to the broker+ // Cannot use authentication or authorisation cache as auto provisioning MQTT clients will authenticate as anonymous and this is then baked into the created ServerSession and cannot be modified+ // so all anonymous sessions will use the same username/password for key lookups in the caches - Can possibly use caching if ActiveMQ makes changes and/or we move to using X.509 TLS with ActiveMQ+ //config.setSecurityInvalidationInterval(600000); // Long cache as we force clear it when neededconfig.setAuthenticationCacheSize(0);config.setAuthorizationCacheSize(0);@@ -270,7 +289,7 @@ public class MQTTBrokerService extends RouteBuilder implements ContainerService,@Overridepublic void configure() throws Exception {from(PERSISTENCE_TOPIC)- .routeId("UserPersistenceChanges")+ .routeId("Persistence-UserAndAssetLink").filter(body().isInstanceOf(PersistenceEvent.class)).process(exchange -> {PersistenceEvent<?> persistenceEvent = (PersistenceEvent<?>) exchange.getIn().getBody(PersistenceEvent.class);@@ -299,6 +318,8 @@ public class MQTTBrokerService extends RouteBuilder implements ContainerService,} else if (persistenceEvent.getEntity() instanceof UserAssetLink userAssetLink) {String userID = userAssetLink.getId().getUserId();// Debounce force disconnect check of this user's sessions as there could be many asset links changing+ List<PersistenceEvent<UserAssetLink>> changedUserAssetLinks = userAssetLinkChangeMap.computeIfAbsent(userID, id -> Collections.synchronizedList(new ArrayList<>()));+ changedUserAssetLinks.add((PersistenceEvent<UserAssetLink>) persistenceEvent);userAssetDisconnectDebouncer.call(userID);}});@@ -337,28 +358,28 @@ public class MQTTBrokerService extends RouteBuilder implements ContainerService,public void connectionFailed(ActiveMQException exception, boolean failedOver, String scaleDownTargetNodeID) {// TODO: Force delete session (don't allow retained/durable sessions)+ connectionIDConnectionMap.remove(getConnectionIDString(connection));++ if (connection.getClientID() != null) {+ RemotingConnection remotingConnection = clientIDConnectionMap.remove(connection.getClientID());+ if (remotingConnection != null) {+ disconnectedConnectionCache.put(connection.getClientID(), remotingConnection);+ }+ }+if (exception.getType() == ActiveMQExceptionType.REMOTE_DISCONNECT) {// Seems to be the type for graceful close of connection// Notify handlers of connection close- LOG.log(DEBUG, () -> "Client connection closed: " + connectionToString(connection));+ LOG.log(DEBUG, () -> "Client disconnected: " + connectionToString(connection));for (MQTTHandler handler : getCustomHandlers()) {handler.onDisconnect(connection);}} else {// Notify handlers of connection failure- LOG.log(DEBUG, () -> "Client connection failed: " + connectionToString(connection));+ LOG.log(DEBUG, () -> "Client disconnected (failure=" + exception.getMessage() + "): " + connectionToString(connection));for (MQTTHandler handler : getCustomHandlers()) {handler.onConnectionLost(connection);}}-- connectionIDConnectionMap.remove(getConnectionIDString(connection));-- if (connection.getClientID() != null) {- RemotingConnection remotingConnection = clientIDConnectionMap.remove(connection.getClientID());- if (remotingConnection != null) {- disconnectedConnectionCache.put(connection.getClientID(), remotingConnection);- }- }}});}@@ -378,7 +399,7 @@ public class MQTTBrokerService extends RouteBuilder implements ContainerService,clientIDConnectionMap.put(remotingConnection.getClientID(), remotingConnection);if (!connectionIDConnectionMap.containsKey(connectionID)) {- LOG.log(DEBUG, () -> "Client connection created: " + connectionToString(remotingConnection));+ LOG.log(DEBUG, () -> "Client connected: " + connectionToString(remotingConnection));connectionIDConnectionMap.put(connectionID, remotingConnection);for (MQTTHandler handler : getCustomHandlers()) {handler.onConnect(remotingConnection);@@ -396,7 +417,6 @@ public class MQTTBrokerService extends RouteBuilder implements ContainerService,for (MQTTHandler handler : getCustomHandlers()) {if (handler.handlesTopic(topic)) {String connectionStr = LOG.isLoggable(DEBUG) ? connectionToString(connection) : null;- LOG.log(TRACE, "Handler has handled subscribe: handler=" + handler.getName() + ", topic=" + topic + ", " + connectionStr);LOG.log(DEBUG, "Client subscribed '" + topicStr + "': " + connectionStr);handler.onSubscribe(connection, topic);break;@@ -410,7 +430,6 @@ public class MQTTBrokerService extends RouteBuilder implements ContainerService,for (MQTTHandler handler : getCustomHandlers()) {if (handler.handlesTopic(topic)) {String connectionStr = LOG.isLoggable(DEBUG) ? connectionToString(connection) : null;- LOG.log(TRACE, "Handler has handled unsubscribe: handler=" + handler.getName() + ", topic=" + topic + ", " + connectionStr);LOG.log(DEBUG, "Client unsubscribed '" + topicStr + "': " + connectionStr);handler.onUnsubscribe(connection, topic);break;@@ -422,7 +441,7 @@ public class MQTTBrokerService extends RouteBuilder implements ContainerService,return customHandlers;}- public void processUserAssetLinkChange(String userID) {+ public void processUserAssetLinkChange(String userID, List<PersistenceEvent<UserAssetLink>> changes) {if (TextUtil.isNullOrEmpty(userID)) {return;}@@ -438,7 +457,7 @@ public class MQTTBrokerService extends RouteBuilder implements ContainerService,userConnections.forEach(connection -> {for (MQTTHandler handler : customHandlers) {connection.setSubject(subject);- handler.onUserAssetLinksChanged(connection);+ handler.onUserAssetLinksChanged(connection, changes);}});}@@ -462,6 +481,7 @@ public class MQTTBrokerService extends RouteBuilder implements ContainerService,protected void doForceDisconnect(RemotingConnection connection) {LOG.log(DEBUG, "Force disconnecting client connection: " + connectionToString(connection));connection.disconnect(false);+ ((SecurityStoreImpl)server.getActiveMQServer().getSecurityStore()).invalidateAuthorizationCache();}public boolean disconnectSession(String sessionID) {diff --git a/manager/src/main/java/org/openremote/manager/mqtt/MQTTHandler.java b/manager/src/main/java/org/openremote/manager/mqtt/MQTTHandler.javaindex dedaee82f..5547b76de 100644--- a/manager/src/main/java/org/openremote/manager/mqtt/MQTTHandler.java+++ b/manager/src/main/java/org/openremote/manager/mqtt/MQTTHandler.java@@ -36,15 +36,18 @@ import org.openremote.manager.event.ClientEventService;import org.openremote.manager.security.ManagerIdentityService;import org.openremote.manager.security.ManagerKeycloakIdentityProvider;import org.openremote.model.Container;+import org.openremote.model.PersistenceEvent;+import org.openremote.model.asset.UserAssetLink;import javax.security.auth.Subject;+import java.util.List;import java.util.Objects;import java.util.Optional;import java.util.Set;import java.util.logging.Level;import java.util.logging.Logger;-import static java.lang.System.Logger.Level.WARNING;+import static java.lang.System.Logger.Level.*;import static org.openremote.manager.mqtt.MQTTBrokerService.LOG;/**@@ -101,15 +104,17 @@ public abstract class MQTTHandler {public void start(Container container) throws Exception {Set<String> publishListenerTopics = getPublishListenerTopics();if (publishListenerTopics != null) {- publishListenerTopics.forEach(this::addPublishConsumer);+ for (String publishListenerTopic : publishListenerTopics) {+ addPublishConsumer(publishListenerTopic);+ }}}- protected void addPublishConsumer(String topic) {+ protected void addPublishConsumer(String topic) throws Exception {try {getLogger().info("Adding publish consumer for topic '" + topic + "': handler=" + getName());String coreTopic = MQTTUtil.convertMqttTopicFilterToCoreAddress(topic, mqttBrokerService.wildcardConfiguration);- mqttBrokerService.internalSession.createQueue(new QueueConfiguration(coreTopic).setRoutingType(RoutingType.ANYCAST).setPurgeOnNoConsumers(true).setAutoCreateAddress(true).setAutoCreated(true));+ mqttBrokerService.internalSession.createQueue(new QueueConfiguration(coreTopic).setRoutingType(RoutingType.MULTICAST).setPurgeOnNoConsumers(true).setAutoCreateAddress(true).setAutoCreated(true));ClientConsumer consumer = mqttBrokerService.internalSession.createConsumer(coreTopic);consumer.setMessageHandler(message -> {Topic publishTopic = Topic.parse(MQTTUtil.convertCoreAddressToMqttTopicFilter(message.getAddress(), mqttBrokerService.wildcardConfiguration));@@ -117,7 +122,7 @@ public abstract class MQTTHandler {RemotingConnection connection = mqttBrokerService.getConnectionFromClientID(clientID);if (connection == null) {- LOG.log(WARNING, "Failed to find connection for connected client so dropping publish to topic '" + topic + "': clientID=" + clientID);+ LOG.log(DEBUG, "Client is no longer connected so dropping publish to topic '" + topic + "': clientID=" + clientID);return;}@@ -126,6 +131,7 @@ public abstract class MQTTHandler {});} catch (ActiveMQException e) {getLogger().log(Level.WARNING, "Failed to create handler consumer for topic '" + topic + "': handler=" + getName(), e);+ throw e;}}@@ -218,7 +224,7 @@ public abstract class MQTTHandler {* Called when {@link org.openremote.model.asset.UserAssetLink}s for a restricted user are changed and that user* has an active connection (subject can be accessed from the connection).*/- public void onUserAssetLinksChanged(RemotingConnection connection) {+ public void onUserAssetLinksChanged(RemotingConnection connection, List<PersistenceEvent<UserAssetLink>> changes) {}/**diff --git a/manager/src/main/java/org/openremote/manager/mqtt/UserAssetProvisioningMQTTHandler.java b/manager/src/main/java/org/openremote/manager/mqtt/UserAssetProvisioningMQTTHandler.javaindex 1afd9ba7b..410169c01 100644--- a/manager/src/main/java/org/openremote/manager/mqtt/UserAssetProvisioningMQTTHandler.java+++ b/manager/src/main/java/org/openremote/manager/mqtt/UserAssetProvisioningMQTTHandler.java@@ -78,7 +78,7 @@ public class UserAssetProvisioningMQTTHandler extends MQTTHandler {public void configure() throws Exception {from(PERSISTENCE_TOPIC)- .routeId("ProvisioningConfigPersistenceChanges")+ .routeId("Persistence-ProvisioningConfig").filter(isPersistenceEventForEntityType(ProvisioningConfig.class)).process(exchange -> {PersistenceEvent<ProvisioningConfig<?,?>> persistenceEvent = (PersistenceEvent<ProvisioningConfig<?,?>>)exchange.getIn().getBody(PersistenceEvent.class);diff --git a/manager/src/main/java/org/openremote/manager/notification/NotificationService.java b/manager/src/main/java/org/openremote/manager/notification/NotificationService.javaindex 17806bb1c..2fc114920 100644--- a/manager/src/main/java/org/openremote/manager/notification/NotificationService.java+++ b/manager/src/main/java/org/openremote/manager/notification/NotificationService.java@@ -144,7 +144,7 @@ public class NotificationService extends RouteBuilder implements ContainerServicpublic void configure() throws Exception {from(NOTIFICATION_QUEUE)- .routeId("NotificationQueueProcessor")+ .routeId("NotificationQueue").doTry().process(exchange -> {Notification notification = exchange.getIn().getBody(Notification.class);diff --git a/manager/src/main/java/org/openremote/manager/notification/PushNotificationHandler.java b/manager/src/main/java/org/openremote/manager/notification/PushNotificationHandler.javaindex 5e11dc942..f81384986 100644--- a/manager/src/main/java/org/openremote/manager/notification/PushNotificationHandler.java+++ b/manager/src/main/java/org/openremote/manager/notification/PushNotificationHandler.java@@ -35,7 +35,6 @@ import org.openremote.model.Container;import org.openremote.model.ContainerService;import org.openremote.model.PersistenceEvent;import org.openremote.model.asset.Asset;-import org.openremote.model.asset.UserAssetLink;import org.openremote.model.asset.impl.ConsoleAsset;import org.openremote.model.console.ConsoleProvider;import org.openremote.model.notification.AbstractNotificationMessage;@@ -56,15 +55,14 @@ import java.io.InputStream;import java.nio.file.Files;import java.nio.file.Paths;import java.util.*;+import java.util.concurrent.ConcurrentHashMap;import java.util.logging.Level;import java.util.logging.Logger;import java.util.stream.Collectors;import java.util.stream.StreamSupport;-import static org.openremote.container.concurrent.GlobalLock.withLock;import static org.openremote.manager.gateway.GatewayService.isNotForGateway;import static org.openremote.model.notification.PushNotificationMessage.TargetType.*;-import static org.openremote.model.security.User.EMAIL_NOTIFICATIONS_DISABLED_ATTRIBUTE;import static org.openremote.model.security.User.PUSH_NOTIFICATIONS_DISABLED_ATTRIBUTE;@SuppressWarnings("deprecation")@@ -80,8 +78,8 @@ public class PushNotificationHandler extends RouteBuilder implements Notificatioprotected AssetStorageService assetStorageService;protected GatewayService gatewayService;protected boolean valid;- protected Map<String, String> consoleFCMTokenMap = new HashMap<>();- protected List<String> fcmTokenBlacklist = new ArrayList<>();+ protected Map<String, String> consoleFCMTokenMap = new ConcurrentHashMap<>();+ protected Set<String> fcmTokenBlacklist = Collections.synchronizedSet(new HashSet<>());@Overridepublic int getPriority() {@@ -154,7 +152,7 @@ public class PushNotificationHandler extends RouteBuilder implements Notificatiopublic void configure() throws Exception {// If any console asset was modified in the database, detect push provider changesfrom(PersistenceService.PERSISTENCE_TOPIC)- .routeId("PushNotificationAssetChanges")+ .routeId("Persistence-PushNotificationConsoleAsset").filter(PersistenceService.isPersistenceEventForEntityType(ConsoleAsset.class)).filter(isNotForGateway(gatewayService)).process(exchange -> {@@ -497,23 +495,18 @@ public class PushNotificationHandler extends RouteBuilder implements Notificatio}protected void processConsoleAssetChange(ConsoleAsset asset, PersistenceEvent<ConsoleAsset> persistenceEvent) {+ String fcmToken = consoleFCMTokenMap.remove(asset.getId());+ if (!TextUtil.isNullOrEmpty(fcmToken)) {+ fcmTokenBlacklist.remove(fcmToken);+ }- withLock(getClass().getSimpleName() + "::processAssetChange", () -> {-- String fcmToken = consoleFCMTokenMap.remove(asset.getId());- if (!TextUtil.isNullOrEmpty(fcmToken)) {- fcmTokenBlacklist.remove(fcmToken);- }-- switch (persistenceEvent.getCause()) {-- case CREATE:- case UPDATE:+ switch (persistenceEvent.getCause()) {- consoleFCMTokenMap.put(asset.getId(), getFcmToken(asset).orElse(null));- break;- }- });+ case CREATE:+ case UPDATE:+ getFcmToken(asset).ifPresent(token -> consoleFCMTokenMap.put(asset.getId(), token));+ break;+ }}protected void handleFcmException(FirebaseMessagingException e) {diff --git a/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java b/manager/src/main/java/org/openremote/manager/rules/RulesEngine.javaindex 4103706ae..2537eb39d 100644--- a/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java+++ b/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java@@ -19,6 +19,9 @@*/package org.openremote.manager.rules;+import io.micrometer.core.instrument.MeterRegistry;+import io.micrometer.core.instrument.Tags;+import io.micrometer.core.instrument.Timer;import org.jeasy.rules.api.Facts;import org.jeasy.rules.api.Rule;import org.jeasy.rules.api.RuleListener;@@ -43,6 +46,7 @@ import org.openremote.model.rules.*;import org.openremote.model.util.TextUtil;import java.util.*;+import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ScheduledExecutorService;import java.util.concurrent.ScheduledFuture;import java.util.concurrent.TimeUnit;@@ -71,8 +75,8 @@ public class RulesEngine<T extends Ruleset> {*/public static final class AssetStateLocationPredicates {- protected final String assetId;- protected final Set<GeofencePredicate> locationPredicates;+ final String assetId;+ final Set<GeofencePredicate> locationPredicates;public AssetStateLocationPredicates(String assetId, Set<GeofencePredicate> locationPredicates) {this.assetId = assetId;@@ -110,8 +114,8 @@ public class RulesEngine<T extends Ruleset> {final protected HistoricDatapoints historicFacade;final protected AssetLocationPredicateProcessor assetLocationPredicatesConsumer;- final protected Map<Long, RulesetDeployment> deployments = new LinkedHashMap<>();- final protected Map<Long, RulesetStatus> deploymentStatusMap = new LinkedHashMap<>();+ final protected Map<Long, RulesetDeployment> deployments = new ConcurrentHashMap<>();+ final protected Map<Long, RulesetStatus> deploymentStatusMap = new ConcurrentHashMap<>();final protected RulesFacts facts;final protected AbstractRulesEngine engine;@@ -123,6 +127,7 @@ public class RulesEngine<T extends Ruleset> {// Only used to optimize toString(), contains the details of this engineprotected String deploymentInfo;+ protected Timer rulesFiringTimer;public RulesEngine(TimerService timerService,RulesService rulesService,@@ -136,7 +141,8 @@ public class RulesEngine<T extends Ruleset> {AssetDatapointService assetDatapointService,AssetPredictedDatapointService assetPredictedDatapointService,RulesEngineId<T> id,- AssetLocationPredicateProcessor assetLocationPredicatesConsumer) {+ AssetLocationPredicateProcessor assetLocationPredicatesConsumer,+ MeterRegistry meterRegistry) {this.timerService = timerService;this.rulesService = rulesService;this.executorService = executorService;@@ -189,6 +195,11 @@ public class RulesEngine<T extends Ruleset> {throw ex;}});++ if (meterRegistry != null) {+ meterRegistry.gauge("or.rules.facts", Tags.of("type", id.getScope().getSimpleName(), "id", getEngineId()), facts, (facts) -> (double) facts.getFactCount());+ rulesFiringTimer = meterRegistry.timer("or.rules.firing", Tags.of("type", id.getScope().getSimpleName(), "id", getEngineId()));+ }}public RulesEngineId<T> getId() {@@ -385,6 +396,7 @@ public class RulesEngine<T extends Ruleset> {if (!running) {return;}+ running = false;LOG.info("Stopping: " + this);if (fireTimer != null) {fireTimer.cancel(true);@@ -396,13 +408,16 @@ public class RulesEngine<T extends Ruleset> {}deployments.values().forEach(this::stopRuleset);- running = false;- if (!systemShutdownInProgress && assetLocationPredicatesConsumer != null) {+ if (systemShutdownInProgress) {+ deployments.clear();+ return;+ }++ if (assetLocationPredicatesConsumer != null) {assetLocationPredicatesConsumer.accept(this, null);}- updateDeploymentInfo();publishRulesEngineStatus();}@@ -425,10 +440,6 @@ public class RulesEngine<T extends Ruleset> {}protected synchronized void stopRuleset(RulesetDeployment deployment) {- if (!running) {- return;- }-if (deployment.getStatus() == DEPLOYED) {deployment.stop(facts);deployment.setStatus(READY);@@ -488,8 +499,25 @@ public class RulesEngine<T extends Ruleset> {// Remove any expired temporary factsfacts.removeExpiredTemporaryFacts();- long executionTotalMillis = 0L;+ long executionTotalMillis = timerService.getCurrentTimeMillis();++ if (rulesFiringTimer != null) {+ rulesFiringTimer.record(this::doFire);+ } else {+ doFire();+ }++ trackLocationPredicates(false);+ executionTotalMillis = (timerService.getCurrentTimeMillis() - executionTotalMillis);+ if (executionTotalMillis > 500) {+ LOG.warning("Rules firing took " + executionTotalMillis + "ms on: " + this);+ } else {+ LOG.fine("Rules firing took " + executionTotalMillis + "ms on: " + this);+ }+ }++ protected void doFire() {for (RulesetDeployment deployment : deployments.values()) {try {@@ -510,7 +538,6 @@ public class RulesEngine<T extends Ruleset> {long startTimestamp = timerService.getCurrentTimeMillis();engine.fire(deployment.getRules(), facts);long executionMillis = (timerService.getCurrentTimeMillis() - startTimestamp);- executionTotalMillis += executionMillis;LOG.fine("Rules deployment '" + deployment.getName() + "' executed in: " + executionMillis + "ms");} else {LOG.fine("Rules deployment '" + deployment.getName() + "' skipped as status is: " + status);@@ -533,14 +560,16 @@ public class RulesEngine<T extends Ruleset> {lastFireTimestamp = timerService.getCurrentTimeMillis();}}+ }- trackLocationPredicates(false);-- if (executionTotalMillis > 500) {- LOG.warning("Rules firing took " + executionTotalMillis + "ms on: " + this);- } else {- LOG.finer("Rules firing took " + executionTotalMillis + "ms on: " + this);+ protected String getEngineId() {+ if (id.scope == GlobalRuleset.class) {+ return "";+ }+ if (id.scope == RealmRuleset.class) {+ return id.realm;}+ return id.realm + ":" + id.assetId;}protected synchronized void notifyAssetStatesChanged(AssetStateChangeEvent event) {@@ -643,6 +672,10 @@ public class RulesEngine<T extends Ruleset> {}protected void publishRulesetStatus(RulesetDeployment deployment) {+ if (!running) {+ return;+ }+Ruleset ruleset = deployment.ruleset;RulesetStatus previousStatus = deploymentStatusMap.get(ruleset.getId());String engineId = id == null ? null : id.getRealm().orElse(id.getAssetId().orElse(null));diff --git a/manager/src/main/java/org/openremote/manager/rules/RulesFacts.java b/manager/src/main/java/org/openremote/manager/rules/RulesFacts.javaindex de149581f..20c87cbbb 100644--- a/manager/src/main/java/org/openremote/manager/rules/RulesFacts.java+++ b/manager/src/main/java/org/openremote/manager/rules/RulesFacts.java@@ -141,6 +141,10 @@ public class RulesFacts extends Facts implements RuleListener {).parallel();}+ public long getFactCount() {+ return getNamedFacts().size() + getAnonymousFacts().size() + getAssetStates().size() + getAssetEvents().size();+ }+@SuppressWarnings("unchecked")public Map<String, Object> getVars() {return get(EXECUTION_VARS);diff --git a/manager/src/main/java/org/openremote/manager/rules/RulesHealthStatusProvider.java b/manager/src/main/java/org/openremote/manager/rules/RulesHealthStatusProvider.javaindex e83b92591..ccf7d57c6 100644--- a/manager/src/main/java/org/openremote/manager/rules/RulesHealthStatusProvider.java+++ b/manager/src/main/java/org/openremote/manager/rules/RulesHealthStatusProvider.java@@ -27,7 +27,7 @@ import org.openremote.model.rules.RealmRuleset;import org.openremote.model.system.HealthStatusProvider;import org.openremote.model.util.ValueUtil;-public class RulesHealthStatusProvider implements HealthStatusProvider, ContainerService {+public class RulesHealthStatusProvider implements HealthStatusProvider {public static final String NAME = "rules";protected RulesService rulesService;diff --git a/manager/src/main/java/org/openremote/manager/rules/RulesService.java b/manager/src/main/java/org/openremote/manager/rules/RulesService.javaindex a183cd5bc..9648d1dd2 100644--- a/manager/src/main/java/org/openremote/manager/rules/RulesService.java+++ b/manager/src/main/java/org/openremote/manager/rules/RulesService.java@@ -19,6 +19,8 @@*/package org.openremote.manager.rules;+import io.micrometer.core.instrument.MeterRegistry;+import jakarta.persistence.EntityManager;import org.apache.camel.builder.RouteBuilder;import org.openremote.container.message.MessageBrokerService;import org.openremote.container.persistence.PersistenceService;@@ -58,7 +60,6 @@ import org.openremote.model.util.TimeUtil;import org.openremote.model.value.MetaHolder;import org.openremote.model.value.MetaItemType;-import jakarta.persistence.EntityManager;import java.util.*;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ConcurrentMap;@@ -69,8 +70,6 @@ import java.util.stream.Collectors;import java.util.stream.Stream;import static java.util.logging.Level.SEVERE;-import static org.openremote.container.concurrent.GlobalLock.withLock;-import static org.openremote.container.concurrent.GlobalLock.withLockReturning;import static org.openremote.container.persistence.PersistenceService.PERSISTENCE_TOPIC;import static org.openremote.container.persistence.PersistenceService.isPersistenceEventForEntityType;import static org.openremote.container.util.MapAccess.getInteger;@@ -116,8 +115,8 @@ public class RulesService extends RouteBuilder implements ContainerService, Assepublic static final String OR_RULES_QUICK_FIRE_MILLIS = "OR_RULES_QUICK_FIRE_MILLIS";public static final int OR_RULES_QUICK_FIRE_MILLIS_DEFAULT = 3000;private static final Logger LOG = Logger.getLogger(RulesService.class.getName());- protected final ConcurrentMap<String, RulesEngine<RealmRuleset>> realmEngines = new ConcurrentHashMap<>();- protected final ConcurrentMap<String, RulesEngine<AssetRuleset>> assetEngines = new ConcurrentHashMap<>();+ protected final Map<String, RulesEngine<RealmRuleset>> realmEngines = new ConcurrentHashMap<>();+ protected final Map<String, RulesEngine<AssetRuleset>> assetEngines = new ConcurrentHashMap<>();protected List<GeofenceAssetAdapter> geofenceAssetAdapters = new ArrayList<>();protected TimerService timerService;protected ScheduledExecutorService executorService;@@ -135,7 +134,7 @@ public class RulesService extends RouteBuilder implements ContainerService, Asseprotected RulesEngine<GlobalRuleset> globalEngine;protected Realm[] realms;protected AssetLocationPredicateProcessor locationPredicateRulesConsumer;- protected ConcurrentMap<RulesEngine<?>, List<RulesEngine.AssetStateLocationPredicates>> engineAssetLocationPredicateMap = new ConcurrentHashMap<>();+ protected final ConcurrentMap<RulesEngine<?>, List<RulesEngine.AssetStateLocationPredicates>> engineAssetLocationPredicateMap = new ConcurrentHashMap<>();protected final Set<String> assetsWithModifiedLocationPredicates = new HashSet<>();// Keep global list of asset states that have been pushed to any engines// The objects are already in memory inside the rule engines but keeping them@@ -147,6 +146,7 @@ public class RulesService extends RouteBuilder implements ContainerService, Asseprotected long quickFireMillis;protected boolean initDone;protected boolean startDone;+ protected MeterRegistry meterRegistry;@Overridepublic int getPriority() {@@ -176,6 +176,8 @@ public class RulesService extends RouteBuilder implements ContainerService, Assereturn;}+ meterRegistry = container.getMeterRegistry();+clientEventService.addSubscriptionAuthorizer((realm, auth, subscription) -> {if (subscription.isEventType(RulesEngineStatusEvent.class) || subscription.isEventType(RulesetChangedEvent.class)) {@@ -234,7 +236,7 @@ public class RulesService extends RouteBuilder implements ContainerService, Assepublic void configure() throws Exception {// If any ruleset was modified in the database then check its' status and undeploy, deploy, or update itfrom(PERSISTENCE_TOPIC)- .routeId("RulesetPersistenceChanges")+ .routeId("Persistence-Ruleset").filter(isPersistenceEventForEntityType(Ruleset.class)).filter(isNotForGateway(gatewayService)).process(exchange -> {@@ -242,10 +244,10 @@ public class RulesService extends RouteBuilder implements ContainerService, AsseprocessRulesetChange((Ruleset) persistenceEvent.getEntity(), persistenceEvent.getCause());});- // If any realm was modified in the database then check its' status and undeploy, deploy or update any+ // If any realm was modified in the database then check its status and undeploy, deploy or update any// associated rulesetsfrom(PERSISTENCE_TOPIC)- .routeId("RuleEngineRealmChanges")+ .routeId("Persistence-RulesRealm").filter(isPersistenceEventForEntityType(Realm.class)).filter(isNotForGateway(gatewayService)).process(exchange -> {@@ -256,7 +258,7 @@ public class RulesService extends RouteBuilder implements ContainerService, Asse// If any asset was modified in the database, detect changed attributesfrom(PERSISTENCE_TOPIC)- .routeId("RuleEngineAssetChanges")+ .routeId("Persistence-RulesAsset").filter(isPersistenceEventForEntityType(Asset.class)).process(exchange -> {PersistenceEvent<Asset<?>> persistenceEvent = (PersistenceEvent<Asset<?>>) exchange.getIn().getBody(PersistenceEvent.class);@@ -338,29 +340,27 @@ public class RulesService extends RouteBuilder implements ContainerService, Asse@Overridepublic void stop(Container container) throws Exception {- withLock(getClass().getSimpleName() + "::stop", () -> {- for (GeofenceAssetAdapter geofenceAssetAdapter : geofenceAssetAdapters) {- try {- geofenceAssetAdapter.stop(container);- } catch (Exception e) {- LOG.log(SEVERE, "Exception thrown whilst stopping geofence adapter", e);- }+ for (GeofenceAssetAdapter geofenceAssetAdapter : geofenceAssetAdapters) {+ try {+ geofenceAssetAdapter.stop(container);+ } catch (Exception e) {+ LOG.log(SEVERE, "Exception thrown whilst stopping geofence adapter", e);}+ }- assetEngines.forEach((assetId, rulesEngine) -> rulesEngine.stop(true));- assetEngines.clear();- realmEngines.forEach((realm, rulesEngine) -> rulesEngine.stop(true));- realmEngines.clear();+ assetEngines.forEach((assetId, rulesEngine) -> rulesEngine.stop(true));+ assetEngines.clear();+ realmEngines.forEach((realm, rulesEngine) -> rulesEngine.stop(true));+ realmEngines.clear();- if (globalEngine != null) {- globalEngine.stop(true);- globalEngine = null;- }+ if (globalEngine != null) {+ globalEngine.stop(true);+ globalEngine = null;+ }- synchronized (assetStates) {- assetStates.clear();- }- });+ synchronized (assetStates) {+ assetStates.clear();+ }for (GeofenceAssetAdapter geofenceAssetAdapter : geofenceAssetAdapters) {geofenceAssetAdapter.stop(container);@@ -455,189 +455,182 @@ public class RulesService extends RouteBuilder implements ContainerService, Asse}protected void processRealmChange(Realm realm, PersistenceEvent.Cause cause) {- withLock(getClass().getSimpleName() + "::processRealmChange", () -> {- // Check if enabled status has changed- boolean wasEnabled = Arrays.stream(realms).anyMatch(t -> realm.getName().equals(t.getName()) && realm.getId().equals(t.getId()));- boolean isEnabled = realm.getEnabled() && cause != PersistenceEvent.Cause.DELETE;- realms = Arrays.stream(identityService.getIdentityProvider().getRealms()).filter(Realm::getEnabled).toArray(Realm[]::new);-- if (wasEnabled == isEnabled) {- // Nothing to do here- return;+ // Check if enabled status has changed+ boolean wasEnabled = Arrays.stream(realms).anyMatch(t -> realm.getName().equals(t.getName()) && realm.getId().equals(t.getId()));+ boolean isEnabled = realm.getEnabled() && cause != PersistenceEvent.Cause.DELETE;+ realms = Arrays.stream(identityService.getIdentityProvider().getRealms()).filter(Realm::getEnabled).toArray(Realm[]::new);++ if (wasEnabled == isEnabled) {+ // Nothing to do here+ return;+ }++ if (wasEnabled) {+ // Remove realm rules engine for this realm if it exists+ RulesEngine<RealmRuleset> realmRulesEngine = realmEngines.get(realm.getName());+ if (realmRulesEngine != null) {+ realmRulesEngine.stop();+ realmEngines.remove(realm.getName());}- if (wasEnabled) {- // Remove realm rules engine for this realm if it exists- RulesEngine<RealmRuleset> realmRulesEngine = realmEngines.get(realm.getName());- if (realmRulesEngine != null) {- realmRulesEngine.stop();- realmEngines.remove(realm.getName());- }+ // Remove any asset rules engines for assets in this realm+ assetEngines.values().stream()+ .filter(re -> re.getId().getRealm().map(r -> r.equals(realm.getName())).orElse(false))+ .forEach(RulesEngine::stop);+ assetEngines.entrySet().removeIf(entry ->+ entry.getValue().getId().getRealm().map(r -> r.equals(realm.getName())).orElse(+ false)+ );- // Remove any asset rules engines for assets in this realm- assetEngines.values().stream()- .filter(re -> re.getId().getRealm().map(r -> r.equals(realm.getName())).orElse(false))- .forEach(RulesEngine::stop);- assetEngines.entrySet().removeIf(entry ->- entry.getValue().getId().getRealm().map(r -> r.equals(realm.getName())).orElse(- false)- );+ } else {+ // Create realm rules engines for this realm if it has any rulesets+ rulesetStorageService+ .findAll(+ RealmRuleset.class,+ new RulesetQuery()+ .setRealm(realm.getName())+ .setFullyPopulate(true)+ .setEnabledOnly(true))+ .stream()+ .map(this::deployRealmRuleset)+ .filter(Objects::nonNull)+ .forEach(RulesEngine::start);++ // Create any asset rules engines for assets in this realm that have rulesets+ deployAssetRulesets(+ rulesetStorageService.findAll(+ AssetRuleset.class,+ new RulesetQuery()+ .setRealm(realm.getName())+ .setEnabledOnly(true)+ .setFullyPopulate(true)))+ .forEach(RulesEngine::start);+ }+ }- } else {- // Create realm rules engines for this realm if it has any rulesets- rulesetStorageService- .findAll(- RealmRuleset.class,- new RulesetQuery()- .setRealm(realm.getName())- .setFullyPopulate(true)- .setEnabledOnly(true))+ protected void processAssetChange(Asset<?> asset, PersistenceEvent<Asset<?>> persistenceEvent) {+ switch (persistenceEvent.getCause()) {+ case CREATE: {+ // Load rule state attributes into rules engine+ asset+ .getAttributes().stream()- .map(this::deployRealmRuleset)- .filter(Objects::nonNull)- .forEach(RulesEngine::start);-- // Create any asset rules engines for assets in this realm that have rulesets- deployAssetRulesets(- rulesetStorageService.findAll(- AssetRuleset.class,- new RulesetQuery()- .setRealm(realm.getName())- .setEnabledOnly(true)- .setFullyPopulate(true)))- .forEach(RulesEngine::start);+ .filter(RulesService::isRuleState)+ .forEach(attribute -> {+ AssetState<?> assetState = new AssetState<>(asset, attribute, Source.INTERNAL);+ LOG.finest("Asset was persisted (" + persistenceEvent.getCause() + "), inserting fact: " + assetState);+ updateAssetState(assetState);+ });+ break;}- });- }+ case UPDATE: {- protected void processAssetChange(Asset<?> asset, PersistenceEvent<Asset<?>> persistenceEvent) {- withLock(getClass().getSimpleName() + "::processAssetChange", () -> {-- switch (persistenceEvent.getCause()) {- case CREATE: {- // Load rule state attributes into rules engine- asset- .getAttributes()- .stream()- .filter(RulesService::isRuleState)- .forEach(attribute -> {- AssetState<?> assetState = new AssetState<>(asset, attribute, Source.INTERNAL);- LOG.finest("Asset was persisted (" + persistenceEvent.getCause() + "), inserting fact: " + assetState);- updateAssetState(assetState);- });- break;- }- case UPDATE: {+ boolean attributesChanged = persistenceEvent.hasPropertyChanged("attributes");+ AttributeMap oldAttributes = attributesChanged ? ((AttributeMap) persistenceEvent.getPreviousState("attributes")) : asset.getAttributes();+ AttributeMap currentAttributes = asset.getAttributes();- boolean attributesChanged = persistenceEvent.hasPropertyChanged("attributes");- AttributeMap oldAttributes = attributesChanged ? ((AttributeMap) persistenceEvent.getPreviousState("attributes")) : asset.getAttributes();- AttributeMap currentAttributes = asset.getAttributes();+ List<Attribute<?>> oldStateAttributes = oldAttributes+ .stream()+ .filter(RulesService::isRuleState).toList();- List<Attribute<?>> oldStateAttributes = oldAttributes- .stream()- .filter(RulesService::isRuleState).toList();+ List<Attribute<?>> newStateAttributes = currentAttributes+ .stream()+ .filter(RulesService::isRuleState).toList();- List<Attribute<?>> newStateAttributes = currentAttributes- .stream()- .filter(RulesService::isRuleState).toList();+ // Just retract all old attributes rather than compare every value that might cause asset state to mutate+ oldStateAttributes.forEach(attribute -> {+ AssetState<?> assetState = new AssetState<>(asset, attribute, Source.INTERNAL);+ LOG.finest("Asset was persisted (" + persistenceEvent.getCause() + "), retracting obsolete fact: " + assetState);+ retractAssetState(assetState);+ });- // Just retract all old attributes rather than compare every value that might cause asset state to mutate- oldStateAttributes.forEach(attribute -> {+ // Insert new states for new or changed attributes+ newStateAttributes.forEach(attribute -> {+ AssetState<?> assetState = new AssetState<>(asset, attribute, Source.INTERNAL);+ LOG.finest("Asset was persisted (" + persistenceEvent.getCause() + "), inserting fact: " + assetState);+ updateAssetState(assetState);+ });+ break;+ }+ case DELETE:+ // Retract any facts that were associated with this asset+ asset.getAttributes().stream()+ .filter(RulesService::isRuleState)+ .forEach(attribute -> {AssetState<?> assetState = new AssetState<>(asset, attribute, Source.INTERNAL);- LOG.finest("Asset was persisted (" + persistenceEvent.getCause() + "), retracting obsolete fact: " + assetState);+ LOG.finest("Asset was persisted (" + persistenceEvent.getCause() + "), retracting fact: " + assetState);retractAssetState(assetState);});-- // Insert new states for new or changed attributes- newStateAttributes.forEach(attribute -> {- AssetState<?> assetState = new AssetState<>(asset, attribute, Source.INTERNAL);- LOG.finest("Asset was persisted (" + persistenceEvent.getCause() + "), inserting fact: " + assetState);- updateAssetState(assetState);- });- break;- }- case DELETE:- // Retract any facts that were associated with this asset- asset.getAttributes().stream()- .filter(RulesService::isRuleState)- .forEach(attribute -> {- AssetState<?> assetState = new AssetState<>(asset, attribute, Source.INTERNAL);- LOG.finest("Asset was persisted (" + persistenceEvent.getCause() + "), retracting fact: " + assetState);- retractAssetState(assetState);- });- break;- }- });+ break;+ }}- protected void processRulesetChange(Ruleset ruleset, PersistenceEvent.Cause cause) {- withLock(getClass().getSimpleName() + "::processRulesetChange", () -> {- if (cause == PersistenceEvent.Cause.DELETE || !ruleset.isEnabled()) {- if (ruleset instanceof GlobalRuleset) {- undeployGlobalRuleset((GlobalRuleset) ruleset);- } else if (ruleset instanceof RealmRuleset) {- undeployRealmRuleset((RealmRuleset) ruleset);- } else if (ruleset instanceof AssetRuleset) {- undeployAssetRuleset((AssetRuleset) ruleset);- }- } else {- if (ruleset instanceof GlobalRuleset) {+ protected synchronized void processRulesetChange(Ruleset ruleset, PersistenceEvent.Cause cause) {+ if (cause == PersistenceEvent.Cause.DELETE || !ruleset.isEnabled()) {+ if (ruleset instanceof GlobalRuleset) {+ undeployGlobalRuleset((GlobalRuleset) ruleset);+ } else if (ruleset instanceof RealmRuleset) {+ undeployRealmRuleset((RealmRuleset) ruleset);+ } else if (ruleset instanceof AssetRuleset) {+ undeployAssetRuleset((AssetRuleset) ruleset);+ }+ } else {+ if (ruleset instanceof GlobalRuleset) {- boolean isNewEngine = globalEngine == null;- RulesEngine<GlobalRuleset> engine = deployGlobalRuleset((GlobalRuleset) ruleset);+ boolean isNewEngine = globalEngine == null;+ RulesEngine<GlobalRuleset> engine = deployGlobalRuleset((GlobalRuleset) ruleset);- if (isNewEngine) {- synchronized (assetStates) {- // Push all existing facts into the engine- assetStates.forEach(assetState -> engine.updateOrInsertAssetState(assetState, true));- }+ if (isNewEngine) {+ synchronized (assetStates) {+ // Push all existing facts into the engine+ assetStates.forEach(assetState -> engine.updateOrInsertAssetState(assetState, true));}+ }- engine.start();+ engine.start();- } else if (ruleset instanceof RealmRuleset) {+ } else if (ruleset instanceof RealmRuleset) {- boolean isNewEngine = !realmEngines.containsKey(((RealmRuleset) ruleset).getRealm());- RulesEngine<RealmRuleset> engine = deployRealmRuleset((RealmRuleset) ruleset);+ boolean isNewEngine = !realmEngines.containsKey(((RealmRuleset) ruleset).getRealm());+ RulesEngine<RealmRuleset> engine = deployRealmRuleset((RealmRuleset) ruleset);- if (isNewEngine) {- // Push all existing facts into the engine- synchronized (assetStates) {- assetStates.forEach(assetState -> {- if (assetState.getRealm().equals(((RealmRuleset) ruleset).getRealm())) {- engine.updateOrInsertAssetState(assetState, true);- }- });- }+ if (isNewEngine) {+ // Push all existing facts into the engine+ synchronized (assetStates) {+ assetStates.forEach(assetState -> {+ if (assetState.getRealm().equals(((RealmRuleset) ruleset).getRealm())) {+ engine.updateOrInsertAssetState(assetState, true);+ }+ });}+ }- engine.start();-- } else if (ruleset instanceof AssetRuleset) {+ engine.start();- // Must reload from the database, the ruleset might not be completely hydrated on CREATE or UPDATE- AssetRuleset assetRuleset = rulesetStorageService.find(AssetRuleset.class, ruleset.getId());- boolean isNewEngine = !assetEngines.containsKey(((AssetRuleset) ruleset).getAssetId());- RulesEngine<AssetRuleset> engine = deployAssetRuleset(assetRuleset);+ } else if (ruleset instanceof AssetRuleset) {- if (isNewEngine) {- // Push all existing facts for this asset (and it's children into the engine)- getAssetStatesInScope(((AssetRuleset) ruleset).getAssetId())- .forEach(assetState -> engine.updateOrInsertAssetState(assetState, true));- }+ // Must reload from the database, the ruleset might not be completely hydrated on CREATE or UPDATE+ AssetRuleset assetRuleset = rulesetStorageService.find(AssetRuleset.class, ruleset.getId());+ boolean isNewEngine = !assetEngines.containsKey(((AssetRuleset) ruleset).getAssetId());+ RulesEngine<AssetRuleset> engine = deployAssetRuleset(assetRuleset);- engine.start();+ if (isNewEngine) {+ // Push all existing facts for this asset (and it's children into the engine)+ getAssetStatesInScope(((AssetRuleset) ruleset).getAssetId())+ .forEach(assetState -> engine.updateOrInsertAssetState(assetState, true));}++ engine.start();}- });+ }}/*** Deploy the ruleset into the global engine creating the engine if necessary.*/protected RulesEngine<GlobalRuleset> deployGlobalRuleset(GlobalRuleset ruleset) {- return withLockReturning(getClass().getSimpleName() + "::deployGlobalRuleset", () -> {+ synchronized (this) {// Global rules have access to everything in the systemif (globalEngine == null) {globalEngine = new RulesEngine<>(@@ -653,34 +646,34 @@ public class RulesService extends RouteBuilder implements ContainerService, AsseassetDatapointService,assetPredictedDatapointService,new RulesEngineId<>(),- locationPredicateRulesConsumer+ locationPredicateRulesConsumer,+ meterRegistry);}+ }- globalEngine.addRuleset(ruleset);+ globalEngine.addRuleset(ruleset);- return globalEngine;- });+ return globalEngine;}- protected void undeployGlobalRuleset(GlobalRuleset ruleset) {- withLock(getClass().getSimpleName() + "::undeployGlobalRuleset", () -> {- if (globalEngine == null) {- return;- }+ protected synchronized void undeployGlobalRuleset(GlobalRuleset ruleset) {+ if (globalEngine == null) {+ return;+ }- if (globalEngine.removeRuleset(ruleset)) {- globalEngine.stop();- globalEngine = null;- }- });+ if (globalEngine.removeRuleset(ruleset)) {+ globalEngine.stop();+ globalEngine = null;+ }}protected RulesEngine<RealmRuleset> deployRealmRuleset(RealmRuleset ruleset) {- return withLockReturning(getClass().getSimpleName() + "::deployRealmRuleset", () -> {+ RulesEngine<RealmRuleset> realmRulesEngine;+ synchronized (this) {// Look for existing rules engines for this realm- RulesEngine<RealmRuleset> realmRulesEngine = realmEngines+ realmRulesEngine = realmEngines.computeIfAbsent(ruleset.getRealm(), (realm) ->new RulesEngine<>(timerService,@@ -695,28 +688,26 @@ public class RulesService extends RouteBuilder implements ContainerService, AsseassetDatapointService,assetPredictedDatapointService,new RulesEngineId<>(realm),- locationPredicateRulesConsumer+ locationPredicateRulesConsumer,+ meterRegistry));+ }- realmRulesEngine.addRuleset(ruleset);+ realmRulesEngine.addRuleset(ruleset);- return realmRulesEngine;- });+ return realmRulesEngine;}- protected void undeployRealmRuleset(RealmRuleset ruleset) {- withLock(getClass().getSimpleName() + "::undeployRealmRuleset", () -> {-- RulesEngine<RealmRuleset> rulesEngine = realmEngines.get(ruleset.getRealm());- if (rulesEngine == null) {- return;- }+ protected synchronized void undeployRealmRuleset(RealmRuleset ruleset) {+ RulesEngine<RealmRuleset> rulesEngine = realmEngines.get(ruleset.getRealm());+ if (rulesEngine == null) {+ return;+ }- if (rulesEngine.removeRuleset(ruleset)) {- rulesEngine.stop();- realmEngines.remove(ruleset.getRealm());- }- });+ if (rulesEngine.removeRuleset(ruleset)) {+ rulesEngine.stop();+ realmEngines.remove(ruleset.getRealm());+ }}protected Stream<RulesEngine<AssetRuleset>> deployAssetRulesets(List<AssetRuleset> rulesets) {@@ -755,10 +746,11 @@ public class RulesService extends RouteBuilder implements ContainerService, Asse}protected RulesEngine<AssetRuleset> deployAssetRuleset(AssetRuleset ruleset) {- return withLockReturning(getClass().getSimpleName() + "::deployAssetRuleset", () -> {+ RulesEngine<AssetRuleset> assetRulesEngine;+ synchronized (this) {// Look for existing rules engine for this asset- RulesEngine<AssetRuleset> assetRulesEngine = assetEngines+ assetRulesEngine = assetEngines.computeIfAbsent(ruleset.getAssetId(), (assetId) ->new RulesEngine<>(timerService,@@ -773,27 +765,25 @@ public class RulesService extends RouteBuilder implements ContainerService, AsseassetDatapointService,assetPredictedDatapointService,new RulesEngineId<>(ruleset.getRealm(), assetId),- locationPredicateRulesConsumer+ locationPredicateRulesConsumer,+ meterRegistry));+ }- assetRulesEngine.addRuleset(ruleset);-- return assetRulesEngine;- });+ assetRulesEngine.addRuleset(ruleset);+ return assetRulesEngine;}- protected void undeployAssetRuleset(AssetRuleset ruleset) {- withLock(getClass().getSimpleName() + "::undeployAssetRuleset", () -> {- RulesEngine<AssetRuleset> rulesEngine = assetEngines.get(ruleset.getAssetId());- if (rulesEngine == null) {- return;- }+ protected synchronized void undeployAssetRuleset(AssetRuleset ruleset) {+ RulesEngine<AssetRuleset> rulesEngine = assetEngines.get(ruleset.getAssetId());+ if (rulesEngine == null) {+ return;+ }- if (rulesEngine.removeRuleset(ruleset)) {- rulesEngine.stop();- assetEngines.remove(ruleset.getAssetId());- }- });+ if (rulesEngine.removeRuleset(ruleset)) {+ rulesEngine.stop();+ assetEngines.remove(ruleset.getAssetId());+ }}protected void insertAssetEvent(AssetState<?> assetState, long expiresMillis) {@@ -902,8 +892,7 @@ public class RulesService extends RouteBuilder implements ContainerService, Asse// All location predicates have been removed so record each asset state as modifiedassetsWithModifiedLocationPredicates.addAll(existingAssetStateLocationPredicates.stream().map(- RulesEngine.AssetStateLocationPredicates::getAssetId).collect(- Collectors.toList()));+ RulesEngine.AssetStateLocationPredicates::getAssetId).toList());// Remove this engine from the mapreturn null;});@@ -916,8 +905,7 @@ public class RulesService extends RouteBuilder implements ContainerService, Asse// All asset states are new so record them all as modifiedassetsWithModifiedLocationPredicates.addAll(newEngineAssetStateLocationPredicates.stream().map(- RulesEngine.AssetStateLocationPredicates::getAssetId).collect(- Collectors.toList()));+ RulesEngine.AssetStateLocationPredicates::getAssetId).toList());} else {// Find obsolete and modified asset statesexistingEngineAssetStateLocationPredicates.forEach(diff --git a/manager/src/main/java/org/openremote/manager/rules/RulesetDeployment.java b/manager/src/main/java/org/openremote/manager/rules/RulesetDeployment.javaindex d9d557b03..cf1fe9666 100644--- a/manager/src/main/java/org/openremote/manager/rules/RulesetDeployment.java+++ b/manager/src/main/java/org/openremote/manager/rules/RulesetDeployment.java@@ -40,10 +40,7 @@ import org.openremote.model.util.Pair;import org.openremote.model.util.ValueUtil;import javax.script.*;-import java.util.ArrayList;-import java.util.Collection;-import java.util.Date;-import java.util.List;+import java.util.*;import java.util.concurrent.Future;import java.util.concurrent.ScheduledExecutorService;import java.util.concurrent.ScheduledFuture;@@ -51,8 +48,6 @@ import java.util.concurrent.TimeUnit;import java.util.logging.Level;import java.util.logging.Logger;-import static org.openremote.container.concurrent.GlobalLock.withLock;-public class RulesetDeployment {/**@@ -133,7 +128,7 @@ public class RulesetDeployment {final protected Webhooks webhooksFacade;final protected HistoricDatapoints historicDatapointsFacade;final protected PredictedDatapoints predictedDatapointsFacade;- final protected List<ScheduledFuture<?>> scheduledRuleActions = new ArrayList<>();+ final protected List<ScheduledFuture<?>> scheduledRuleActions = Collections.synchronizedList(new ArrayList<>());protected RulesetStatus status = RulesetStatus.READY;protected Throwable error;protected JsonRulesBuilder jsonRulesBuilder;@@ -252,10 +247,12 @@ public class RulesetDeployment {* during execution*/public void stop(RulesFacts facts) {- scheduledRuleActions.removeIf(scheduledFuture -> {- scheduledFuture.cancel(true);- return true;- });+ synchronized (scheduledRuleActions) {+ scheduledRuleActions.removeIf(scheduledFuture -> {+ scheduledFuture.cancel(true);+ return true;+ });+ }if (jsonRulesBuilder != null) {jsonRulesBuilder.stop(facts);@@ -269,14 +266,11 @@ public class RulesetDeployment {}protected void scheduleRuleAction(Runnable action, long delayMillis) {- withLock(toString() + "::scheduleRuleAction", () -> {- ScheduledFuture<?> future = executorService.schedule(() ->- withLock(toString() + "::scheduledRuleActionFire", () -> {- scheduledRuleActions.removeIf(Future::isDone);- action.run();- }), delayMillis, TimeUnit.MILLISECONDS);- scheduledRuleActions.add(future);- });+ ScheduledFuture<?> future = executorService.schedule(() -> {+ scheduledRuleActions.removeIf(Future::isDone);+ action.run();+ }, delayMillis, TimeUnit.MILLISECONDS);+ scheduledRuleActions.add(future);}protected boolean compileRulesJson(Ruleset ruleset) {diff --git a/manager/src/main/java/org/openremote/manager/rules/facade/NotificationsFacade.java b/manager/src/main/java/org/openremote/manager/rules/facade/NotificationsFacade.javaindex 54d0fcb84..096d2c777 100644--- a/manager/src/main/java/org/openremote/manager/rules/facade/NotificationsFacade.java+++ b/manager/src/main/java/org/openremote/manager/rules/facade/NotificationsFacade.java@@ -41,6 +41,7 @@ public class NotificationsFacade<T extends Ruleset> extends Notifications {this.notificationService = notificationService;}+ // TODO: Priority NotificationService.sendNotification can throw ExchangeTimedOutException: The OUT message was not received within: 10000 millispublic void send(Notification notification) {Notification.Source source;String sourceId = null;diff --git a/manager/src/main/java/org/openremote/manager/rules/geofence/ORConsoleGeofenceAssetAdapter.java b/manager/src/main/java/org/openremote/manager/rules/geofence/ORConsoleGeofenceAssetAdapter.javaindex cd6ab7e99..eaebbce91 100644--- a/manager/src/main/java/org/openremote/manager/rules/geofence/ORConsoleGeofenceAssetAdapter.java+++ b/manager/src/main/java/org/openremote/manager/rules/geofence/ORConsoleGeofenceAssetAdapter.java@@ -118,7 +118,7 @@ public class ORConsoleGeofenceAssetAdapter extends RouteBuilder implements Geofe// If any console asset was modified in the database, detect geofence provider changesfrom(PERSISTENCE_TOPIC)- .routeId("ORConsoleGeofenceAdapterAssetChanges")+ .routeId("Persistence-GeofenceAdapterConsoleAsset").filter(isPersistenceEventForEntityType(ConsoleAsset.class)).filter(isNotForGateway(gatewayService)).process(exchange -> {@@ -262,7 +262,7 @@ public class ORConsoleGeofenceAssetAdapter extends RouteBuilder implements Geofenotification.setTargets(subTargets);executorService.schedule(() -> {- LOG.fine("Notifiying consoles that geofences have changed: " + notification.getTargets());+ LOG.fine("Notifying consoles that geofences have changed: " + notification.getTargets());notificationService.sendNotification(notification);}, (long) i * NOTIFY_ASSETS_BATCH_MILLIS, TimeUnit.MILLISECONDS);});diff --git a/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.java b/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.javaindex 23b9565e6..049a155c3 100644--- a/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.java+++ b/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.java@@ -34,7 +34,6 @@ import org.keycloak.admin.client.resource.UserResource;import org.keycloak.admin.client.resource.*;import org.keycloak.common.enums.SslRequired;import org.keycloak.representations.idm.*;-import org.openremote.container.concurrent.GlobalLock;import org.openremote.container.message.MessageBrokerService;import org.openremote.container.persistence.PersistenceService;import org.openremote.container.security.AuthContext;@@ -938,42 +937,39 @@ public class ManagerKeycloakIdentityProvider extends KeycloakIdentityProvider imthrow new NotFoundException("Realm does not exist: " + realmName);}- // TODO: This should be de-centralised and use a realm lock to prevent any resources being added to the realm during deletion- GlobalLock.withLock("RealmDeletion", () -> {- persistenceService.doTransaction(entityManager -> {+ persistenceService.doTransaction(entityManager -> {- // Delete gateway connections- Query query = entityManager.createQuery("delete from " + GatewayConnection.class.getSimpleName() + " gc " +- "where gc.localRealm = ?1");+ // Delete gateway connections+ Query query = entityManager.createQuery("delete from " + GatewayConnection.class.getSimpleName() + " gc " ++ "where gc.localRealm = ?1");- query.setParameter(1, realmName);- query.executeUpdate();+ query.setParameter(1, realmName);+ query.executeUpdate();- // Delete provisioning configs- query = entityManager.createQuery("delete from " + ProvisioningConfig.class.getSimpleName() + " pc " +- "where pc.realm = ?1");+ // Delete provisioning configs+ query = entityManager.createQuery("delete from " + ProvisioningConfig.class.getSimpleName() + " pc " ++ "where pc.realm = ?1");- query.setParameter(1, realmName);- query.executeUpdate();+ query.setParameter(1, realmName);+ query.executeUpdate();- // Delete Rules- query = entityManager.createQuery("delete from " + RealmRuleset.class.getSimpleName() + " rs " +- "where rs.realm = ?1");- query.setParameter(1, realmName);- query.executeUpdate();+ // Delete Rules+ query = entityManager.createQuery("delete from " + RealmRuleset.class.getSimpleName() + " rs " ++ "where rs.realm = ?1");+ query.setParameter(1, realmName);+ query.executeUpdate();- // Delete Assets- List<String> assetIds = assetStorageService.findAll(new AssetQuery().select(new AssetQuery.Select().excludeAttributes()).realm(new RealmPredicate(realmName))).stream().map(Asset::getId).toList();- assetStorageService.delete(assetIds);- });+ // Delete Assets+ List<String> assetIds = assetStorageService.findAll(new AssetQuery().select(new AssetQuery.Select().excludeAttributes()).realm(new RealmPredicate(realmName))).stream().map(Asset::getId).toList();+ assetStorageService.delete(assetIds);+ });- LOG.fine("Deleting realm: " + realmName);- getRealms(realmsResource -> {- realmsResource.realm(realmName).remove();- return null;- });- persistenceService.publishPersistenceEvent(PersistenceEvent.Cause.DELETE, null, realm, Realm.class, null, null);+ LOG.fine("Deleting realm: " + realmName);+ getRealms(realmsResource -> {+ realmsResource.realm(realmName).remove();+ return null;});+ persistenceService.publishPersistenceEvent(PersistenceEvent.Cause.DELETE, null, realm, Realm.class, null, null);}public ClientRepresentation generateOpenRemoteClientRepresentation() {diff --git a/manager/src/main/java/org/openremote/manager/simulator/SimulatorService.java b/manager/src/main/java/org/openremote/manager/simulator/SimulatorService.javaindex 1d23cfeeb..40e42cb49 100644--- a/manager/src/main/java/org/openremote/manager/simulator/SimulatorService.java+++ b/manager/src/main/java/org/openremote/manager/simulator/SimulatorService.java@@ -36,10 +36,8 @@ import org.openremote.model.simulator.SimulatorState;import java.util.List;import java.util.logging.Logger;-import java.util.stream.Collectors;-import static org.openremote.container.concurrent.GlobalLock.withLockReturning;-import static org.openremote.manager.event.ClientEventService.CLIENT_EVENT_TOPIC;+import static org.openremote.manager.event.ClientEventService.CLIENT_INBOUND_QUEUE;import static org.openremote.manager.event.ClientEventService.getSessionKey;/**@@ -109,8 +107,8 @@ public class SimulatorService extends RouteBuilder implements ContainerService {@Overridepublic void configure() throws Exception {- from(CLIENT_EVENT_TOPIC)- .routeId("FromClientSimulatorRequests")+ from(CLIENT_INBOUND_QUEUE)+ .routeId("ClientInbound-ReadSimulatorState").filter(body().isInstanceOf(RequestSimulatorState.class)).process(exchange -> {RequestSimulatorState event = exchange.getIn().getBody(RequestSimulatorState.class);@@ -128,13 +126,11 @@ public class SimulatorService extends RouteBuilder implements ContainerService {Protocol<?> protocol = agentService.getProtocolInstance(agentId);- if (!(protocol instanceof SimulatorProtocol)) {+ if (!(protocol instanceof SimulatorProtocol simulatorProtocol)) {LOG.warning("Failed to publish simulator state, agent is not a simulator agent: Agent ID=" + agentId);return;}- SimulatorProtocol simulatorProtocol = (SimulatorProtocol)protocol;-SimulatorState simulatorState = getSimulatorState(simulatorProtocol);if (sessionKey != null) {@@ -148,28 +144,25 @@ public class SimulatorService extends RouteBuilder implements ContainerService {* Get info about all attributes linked to this instance (for frontend usage)*/protected SimulatorState getSimulatorState(SimulatorProtocol protocolInstance) {- return withLockReturning(protocolInstance.getProtocolInstanceUri() + "::getSimulatorInfo", () -> {- LOG.info("Getting simulator info for protocol instance: " + protocolInstance);+ LOG.info("Getting simulator info for protocol instance: " + protocolInstance);- // We need asset names instead of identifiers for user-friendly display- List<String> linkedAssetIds = protocolInstance.getLinkedAttributes().keySet().stream().map(AttributeRef::getId).distinct().collect(Collectors.toList());- List<String> assetNames = assetStorageService.findNames(linkedAssetIds.toArray(new String[0]));+ // We need asset names instead of identifiers for user-friendly display+ List<String> linkedAssetIds = protocolInstance.getLinkedAttributes().keySet().stream().map(AttributeRef::getId).distinct().toList();+ List<String> assetNames = assetStorageService.findNames(linkedAssetIds.toArray(new String[0]));- if (assetNames.size() != linkedAssetIds.size()) {- LOG.warning("Retrieved asset names don't match requested asset IDs");- return null;- }+ if (assetNames.size() != linkedAssetIds.size()) {+ LOG.warning("Retrieved asset names don't match requested asset IDs");+ return null;+ }- SimulatorAttributeInfo[] attributeInfos = protocolInstance.getLinkedAttributes().entrySet().stream().map(refAttributeEntry -> {- String assetName = assetNames.get(linkedAssetIds.indexOf(refAttributeEntry.getKey().getId()));- return new SimulatorAttributeInfo(assetName, refAttributeEntry.getKey().getId(), refAttributeEntry.getValue(), protocolInstance.getReplayMap().containsKey(refAttributeEntry.getKey()));- }).toArray(SimulatorAttributeInfo[]::new);+ SimulatorAttributeInfo[] attributeInfos = protocolInstance.getLinkedAttributes().entrySet().stream().map(refAttributeEntry -> {+ String assetName = assetNames.get(linkedAssetIds.indexOf(refAttributeEntry.getKey().getId()));+ return new SimulatorAttributeInfo(assetName, refAttributeEntry.getKey().getId(), refAttributeEntry.getValue(), protocolInstance.getReplayMap().containsKey(refAttributeEntry.getKey()));+ }).toArray(SimulatorAttributeInfo[]::new);- return new SimulatorState(protocolInstance.getAgent().getId(), attributeInfos);- });+ return new SimulatorState(protocolInstance.getAgent().getId(), attributeInfos);}-@Overridepublic String toString() {return getClass().getSimpleName() + "{}";diff --git a/manager/src/main/java/org/openremote/manager/system/CamelHealthStatusProvider.java b/manager/src/main/java/org/openremote/manager/system/CamelHealthStatusProvider.javaindex 765223d6e..4c7b9efee 100644--- a/manager/src/main/java/org/openremote/manager/system/CamelHealthStatusProvider.java+++ b/manager/src/main/java/org/openremote/manager/system/CamelHealthStatusProvider.java@@ -35,7 +35,7 @@ import org.openremote.model.util.ValueUtil;import java.util.HashMap;import java.util.Map;-public class CamelHealthStatusProvider implements HealthStatusProvider, ContainerService {+public class CamelHealthStatusProvider implements HealthStatusProvider {public static final String NAME = "camel";protected MessageBrokerService brokerService;diff --git a/manager/src/main/java/org/openremote/manager/system/HealthService.java b/manager/src/main/java/org/openremote/manager/system/HealthService.javanew file mode 100644index 000000000..8d1c23d9f--- /dev/null+++ b/manager/src/main/java/org/openremote/manager/system/HealthService.java@@ -0,0 +1,199 @@+/*+ * Copyright 2017, OpenRemote Inc.+ *+ * See the CONTRIBUTORS.txt file in the distribution for a+ * full listing of individual contributors.+ *+ * This program is free software: you can redistribute it and/or modify+ * it under the terms of the GNU Affero General Public License as+ * published by the Free Software Foundation, either version 3 of the+ * License, or (at your option) any later version.+ *+ * This program is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+ * GNU Affero General Public License for more details.+ *+ * You should have received a copy of the GNU Affero General Public License+ * along with this program. If not, see <http://www.gnu.org/licenses/>.+ */+package org.openremote.manager.system;++import io.micrometer.core.instrument.Tags;+import io.prometheus.client.exporter.HTTPServer;+import org.apache.camel.CamelContext;+import org.apache.camel.NamedNode;+import org.apache.camel.Route;+import org.apache.camel.component.micrometer.routepolicy.MicrometerRoutePolicy;+import org.apache.camel.component.micrometer.routepolicy.MicrometerRoutePolicyFactory;+import org.apache.camel.component.micrometer.routepolicy.MicrometerRoutePolicyNamingStrategy;+import org.apache.camel.component.seda.SedaEndpoint;+import org.apache.camel.spi.RoutePolicy;+import org.openremote.container.message.MessageBrokerService;+import org.openremote.container.util.MapAccess;+import org.openremote.manager.web.ManagerWebService;+import org.openremote.model.Container;+import org.openremote.model.ContainerService;+import org.openremote.model.system.HealthStatusProvider;+import org.openremote.model.system.StatusResource;++import java.util.ArrayList;+import java.util.List;+import java.util.ServiceLoader;++import static org.apache.camel.component.micrometer.MicrometerConstants.ROUTE_ID_TAG;++/**+ * This service is here to initialise the discovered {@link HealthStatusProvider}s and ({@link StatusResource}+ */+public class HealthService implements ContainerService {++ /**+ * Customised {@link MicrometerRoutePolicy} to add SEDA queue metrics+ */+ protected static class SedaMicrometerRoutePolicy extends MicrometerRoutePolicy {+ @Override+ public void onInit(Route route) {+ if (route.getEndpoint() instanceof SedaEndpoint sedaEndpoint) {+ getMeterRegistry().gauge("or.camel.route.queue.size", getNamingStrategy().getTags(route), sedaEndpoint, SedaEndpoint::getCurrentQueueSize);+ }+ super.onInit(route);+ }+ }++ public static final System.Logger LOG = System.getLogger(HealthService.class.getName());+ public static final String OR_METRICS_PORT = "OR_METRICS_PORT";+ public static final int OR_METRICS_PORT_DEFAULT = 8404;+ protected List<HealthStatusProvider> healthStatusProviderList = new ArrayList<>();+ protected boolean metricsEnabled;+ protected HTTPServer metricsServer;++ @Override+ public int getPriority() {+ return ContainerService.DEFAULT_PRIORITY;+ }++ @Override+ public void init(Container container) throws Exception {++ metricsEnabled = container.getMeterRegistry() != null;+ ServiceLoader.load(HealthStatusProvider.class).forEach(hsp -> {+ LOG.log(System.Logger.Level.INFO, "Health Status Provider: " + hsp.getClass());+ healthStatusProviderList.add(hsp);+ });++ for (HealthStatusProvider healthStatusProvider : healthStatusProviderList) {+ healthStatusProvider.init(container);+ }++ container.getService(ManagerWebService.class).addApiSingleton(+ new StatusResourceImpl(container, healthStatusProviderList)+ );+++ if (metricsEnabled) {+ MessageBrokerService brokerService = container.getService(MessageBrokerService.class);+ int metricsPort = MapAccess.getInteger(container.getConfig(), OR_METRICS_PORT, OR_METRICS_PORT_DEFAULT);+ LOG.log(System.Logger.Level.INFO, "Metrics collection enabled");++ // Add additional web server for metrics (this keeps CI/CD prometheus scraper config simple with no oauth+ // this port can be exposed to the host but not to the public+ metricsServer = new HTTPServer.Builder()+ .withPort(metricsPort)+ .withExecutorService(container.getExecutorService())+ .build();++ // Alternative servlet option to run on existing undertow but would require oauth on prometheus scraper+// DeploymentInfo prometheusServlet = Servlets.deployment()+// .setClassLoader(Container.class.getClassLoader())+// .setContextPath(METRICS_PATH)+// .setDeploymentName("Metrics")+// .addServlets(+// Servlets.servlet("MetricsServlet", MetricsServlet.class)+// .addMapping("/*")+// .setServletSecurityInfo(+// new ServletSecurityInfo().addRoleAllowed(Constants.READ_ADMIN_ROLE)+// )+// );+// HttpHandler handler = WebService.addServletDeployment(container, prometheusServlet, true);+// container.getService(ManagerWebService.class).getRequestHandlers().add(0, new WebService.RequestHandler("Prometheus Metrics", exchange -> exchange.getRequestPath().equals(METRICS_PATH), handler));++ // Configure camel metrics collection here as need access to the custom registry+ MicrometerRoutePolicyFactory micrometerRoutePolicyFactory = new MicrometerRoutePolicyFactory() {+ @Override+ public RoutePolicy createRoutePolicy(CamelContext camelContext, String routeId, NamedNode routeDefinition) {+ SedaMicrometerRoutePolicy answer = new SedaMicrometerRoutePolicy();+ answer.setMeterRegistry(getMeterRegistry());+ answer.setPrettyPrint(isPrettyPrint());+ answer.setDurationUnit(getDurationUnit());+ answer.setNamingStrategy(getNamingStrategy());+ return answer;+ }+ };+ micrometerRoutePolicyFactory.setNamingStrategy(new MicrometerRoutePolicyNamingStrategy() {+ @Override+ public String getName(Route route) {+ return "or_camel_route";+ }++ @Override+ public String getExchangesSucceededName(Route route) {+ return "or_camel_route_succeeded";+ }++ @Override+ public String getExchangesFailedName(Route route) {+ return "or_camel_route_failed";+ }++ @Override+ public String getExchangesTotalName(Route route) {+ return "or_camel_route_total";+ }++ @Override+ public String getFailuresHandledName(Route route) {+ return "or_camel_route_failed_handled";+ }++ @Override+ public String getExternalRedeliveriesName(Route route) {+ return "or_camel_route_redeliveries";+ }++ @Override+ public Tags getTags(Route route) {+ return Tags.of(+ ROUTE_ID_TAG, route.getId());+ }++ @Override+ public Tags getExchangeStatusTags(Route route) {+ return Tags.of(+ ROUTE_ID_TAG, route.getId());+ }+ });+ micrometerRoutePolicyFactory.setMeterRegistry(container.getMeterRegistry());+ brokerService.getContext().addRoutePolicyFactory(micrometerRoutePolicyFactory);+ } else {+ LOG.log(System.Logger.Level.INFO, "Metrics collection disabled");+ }+ }++ @Override+ public void start(Container container) throws Exception {+ for (HealthStatusProvider healthStatusProvider : healthStatusProviderList) {+ healthStatusProvider.start(container);+ }+ }++ @Override+ public void stop(Container container) throws Exception {+ if (metricsServer != null) {+ metricsServer.close();+ }+ for (HealthStatusProvider healthStatusProvider : healthStatusProviderList) {+ healthStatusProvider.stop(container);+ }+ }+}diff --git a/manager/src/main/java/org/openremote/manager/system/HealthStatusService.java b/manager/src/main/java/org/openremote/manager/system/HealthStatusService.javadeleted file mode 100644index acefb4446..000000000--- a/manager/src/main/java/org/openremote/manager/system/HealthStatusService.java+++ /dev/null@@ -1,77 +0,0 @@-/*- * Copyright 2017, OpenRemote Inc.- *- * See the CONTRIBUTORS.txt file in the distribution for a- * full listing of individual contributors.- *- * This program is free software: you can redistribute it and/or modify- * it under the terms of the GNU Affero General Public License as- * published by the Free Software Foundation, either version 3 of the- * License, or (at your option) any later version.- *- * This program is distributed in the hope that it will be useful,- * but WITHOUT ANY WARRANTY; without even the implied warranty of- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the- * GNU Affero General Public License for more details.- *- * You should have received a copy of the GNU Affero General Public License- * along with this program. If not, see <http://www.gnu.org/licenses/>.- */-package org.openremote.manager.system;--import org.openremote.model.Container;-import org.openremote.model.ContainerService;-import org.openremote.manager.web.ManagerWebService;-import org.openremote.model.system.HealthStatusProvider;-import org.openremote.model.system.StatusResource;--import java.util.ArrayList;-import java.util.List;-import java.util.ServiceLoader;--/**- * This service is here to initialise the {@link StatusResource}.- */-public class HealthStatusService implements ContainerService {-- protected List<HealthStatusProvider> healthStatusProviderList = new ArrayList<>();-- @Override- public int getPriority() {- return ContainerService.DEFAULT_PRIORITY;- }-- @Override- public void init(Container container) throws Exception {-- ServiceLoader.load(HealthStatusProvider.class).forEach(healthStatusProviderList::add);-- for (HealthStatusProvider healthStatusProvider : healthStatusProviderList) {- if (healthStatusProvider instanceof ContainerService) {- ((ContainerService) healthStatusProvider).init(container);- }- }-- container.getService(ManagerWebService.class).addApiSingleton(- new StatusResourceImpl(container, healthStatusProviderList)- );- }-- @Override- public void start(Container container) throws Exception {- for (HealthStatusProvider healthStatusProvider : healthStatusProviderList) {- if (healthStatusProvider instanceof ContainerService) {- ((ContainerService) healthStatusProvider).start(container);- }- }- }-- @Override- public void stop(Container container) throws Exception {- for (HealthStatusProvider healthStatusProvider : healthStatusProviderList) {- if (healthStatusProvider instanceof ContainerService) {- ((ContainerService) healthStatusProvider).stop(container);- }- }- }-}diff --git a/manager/src/main/java/org/openremote/manager/system/JvmHealthStatusProvider.java b/manager/src/main/java/org/openremote/manager/system/JvmHealthStatusProvider.javaindex 9fc8f785f..b47f31d33 100644--- a/manager/src/main/java/org/openremote/manager/system/JvmHealthStatusProvider.java+++ b/manager/src/main/java/org/openremote/manager/system/JvmHealthStatusProvider.java@@ -30,7 +30,7 @@ import java.lang.management.MemoryMXBean;import java.lang.management.RuntimeMXBean;import java.lang.management.ThreadMXBean;-public class JvmHealthStatusProvider implements HealthStatusProvider, ContainerService {+public class JvmHealthStatusProvider implements HealthStatusProvider {public static final String NAME = "jvm";diff --git a/manager/src/main/java/org/openremote/manager/system/SslHealthStatusProvider.java b/manager/src/main/java/org/openremote/manager/system/SslHealthStatusProvider.javaindex e5b57f028..e60fbd130 100644--- a/manager/src/main/java/org/openremote/manager/system/SslHealthStatusProvider.java+++ b/manager/src/main/java/org/openremote/manager/system/SslHealthStatusProvider.java@@ -39,7 +39,7 @@ import static java.time.temporal.ChronoUnit.DAYS;import static org.openremote.container.util.MapAccess.getInteger;import static org.openremote.container.util.MapAccess.getString;-public class SslHealthStatusProvider implements X509TrustManager, HealthStatusProvider, ContainerService {+public class SslHealthStatusProvider implements X509TrustManager, HealthStatusProvider {public static final String NAME = "ssl";protected static final Logger LOG = Logger.getLogger(SslHealthStatusProvider.class.getName());diff --git a/manager/src/main/java/org/openremote/manager/system/StatusResourceImpl.java b/manager/src/main/java/org/openremote/manager/system/StatusResourceImpl.javaindex 7504516ea..2356029ca 100644--- a/manager/src/main/java/org/openremote/manager/system/StatusResourceImpl.java+++ b/manager/src/main/java/org/openremote/manager/system/StatusResourceImpl.java@@ -20,7 +20,6 @@package org.openremote.manager.system;import com.fasterxml.jackson.databind.node.ObjectNode;-import org.openremote.container.security.IdentityService;import org.openremote.manager.security.ManagerIdentityService;import org.openremote.model.Container;import org.openremote.model.system.HealthStatusProvider;diff --git a/manager/src/main/java/org/openremote/manager/system/SystemHealthStatusProvider.java b/manager/src/main/java/org/openremote/manager/system/SystemHealthStatusProvider.javaindex 5b9575da3..36c794a66 100644--- a/manager/src/main/java/org/openremote/manager/system/SystemHealthStatusProvider.java+++ b/manager/src/main/java/org/openremote/manager/system/SystemHealthStatusProvider.java@@ -28,7 +28,7 @@ import org.openremote.model.util.ValueUtil;import java.io.File;import java.lang.management.ManagementFactory;-public class SystemHealthStatusProvider implements HealthStatusProvider, ContainerService {+public class SystemHealthStatusProvider implements HealthStatusProvider {public static final String NAME = "system";diff --git a/manager/src/main/resources/META-INF/services/org.openremote.model.ContainerService b/manager/src/main/resources/META-INF/services/org.openremote.model.ContainerServiceindex 84959e501..8fd6dfe33 100644--- a/manager/src/main/resources/META-INF/services/org.openremote.model.ContainerService+++ b/manager/src/main/resources/META-INF/services/org.openremote.model.ContainerService@@ -11,7 +11,7 @@ org.openremote.manager.datapoint.AssetDatapointServiceorg.openremote.manager.asset.AttributeLinkingServiceorg.openremote.manager.asset.AssetProcessingServiceorg.openremote.container.message.MessageBrokerService-org.openremote.manager.system.HealthStatusService+org.openremote.manager.system.HealthServiceorg.openremote.manager.agent.AgentServiceorg.openremote.manager.simulator.SimulatorServiceorg.openremote.manager.map.MapServicediff --git a/manager/src/main/resources/logging-audit.properties b/manager/src/main/resources/logging-audit.propertiesindex 58c76d92e..93880c5df 100644--- a/manager/src/main/resources/logging-audit.properties+++ b/manager/src/main/resources/logging-audit.properties@@ -49,7 +49,6 @@ java.util.logging.FileHandler.append=trueorg.openremote.level=INFO# Processing time logging-#org.openremote.container.concurrent.GlobalLock.level=FINE#org.openremote.manager.asset.AssetProcessingService.level=FINE#org.openremote.manager.asset.AssetStorageService.level=FINE#org.openremote.manager.rules.level=FINEdiff --git a/manager/src/main/resources/logging-dev.properties b/manager/src/main/resources/logging-dev.propertiesindex dafaf3bd5..31d8d719f 100644--- a/manager/src/main/resources/logging-dev.properties+++ b/manager/src/main/resources/logging-dev.properties@@ -32,7 +32,6 @@ org.openremote.level=FINEorg.openremote.agent.protocol.level=FINEST# Processing time logging-#org.openremote.container.concurrent.GlobalLock.level=FINE#org.openremote.manager.asset.AssetProcessingService.level=FINE#org.openremote.manager.asset.AssetStorageService.level=FINE#org.openremote.manager.rules.level=FINEdiff --git a/manager/src/main/resources/logging-timings-audit.properties b/manager/src/main/resources/logging-timings-audit.propertiesindex 1df0c36ac..db5f61688 100644--- a/manager/src/main/resources/logging-timings-audit.properties+++ b/manager/src/main/resources/logging-timings-audit.properties@@ -49,7 +49,6 @@ java.util.logging.FileHandler.append=trueorg.openremote.level=INFO# Processing time logging-org.openremote.container.concurrent.GlobalLock.level=FINEorg.openremote.manager.asset.AssetProcessingService.level=FINEorg.openremote.manager.asset.AssetStorageService.level=FINEorg.openremote.manager.rules.level=FINEdiff --git a/manager/src/main/resources/logging-timings.properties b/manager/src/main/resources/logging-timings.propertiesindex 54fc983fa..bd8c711df 100644--- a/manager/src/main/resources/logging-timings.properties+++ b/manager/src/main/resources/logging-timings.properties@@ -28,9 +28,9 @@ java.util.logging.FileHandler.level=FINE# Output file and formatjava.util.logging.FileHandler.pattern=/deployment/openremote.logjava.util.logging.FileHandler.formatter=org.openremote.container.util.LogFormatter-# Rotate between 10 log files-java.util.logging.FileHandler.count=10-# 10 mb per file+# Rotate N log files+java.util.logging.FileHandler.count=100+# Size per file (bytes)java.util.logging.FileHandler.limit=10000000# Set to false to clear log on restartjava.util.logging.FileHandler.append=true@@ -49,7 +49,6 @@ java.util.logging.FileHandler.append=trueorg.openremote.level=INFO# Processing time logging-org.openremote.container.concurrent.GlobalLock.level=FINEorg.openremote.manager.asset.AssetProcessingService.level=FINEorg.openremote.manager.asset.AssetStorageService.level=FINEorg.openremote.manager.rules.level=FINEdiff --git a/manager/src/main/resources/logging-trace.properties b/manager/src/main/resources/logging-trace.propertiesindex 141fed53f..81fe2c7be 100644--- a/manager/src/main/resources/logging-trace.properties+++ b/manager/src/main/resources/logging-trace.properties@@ -49,7 +49,6 @@ java.util.logging.FileHandler.append=trueorg.openremote.level=FINEST# Processing time logging-#org.openremote.container.concurrent.GlobalLock.level=FINE#org.openremote.manager.asset.AssetProcessingService.level=FINE#org.openremote.manager.asset.AssetStorageService.level=FINE#org.openremote.manager.rules.level=FINEdiff --git a/manager/src/main/resources/logging.properties b/manager/src/main/resources/logging.propertiesindex 897ba560e..25fd846f7 100644--- a/manager/src/main/resources/logging.properties+++ b/manager/src/main/resources/logging.properties@@ -53,7 +53,6 @@ java.util.logging.FileHandler.append=trueorg.openremote.level=INFO# Processing time logging-#org.openremote.container.concurrent.GlobalLock.level=FINE#org.openremote.manager.asset.AssetProcessingService.level=FINE#org.openremote.manager.asset.AssetStorageService.level=FINE#org.openremote.manager.rules.level=FINEdiff --git a/model/build.gradle b/model/build.gradleindex 427c07a93..b76b155a3 100644--- a/model/build.gradle+++ b/model/build.gradle@@ -33,6 +33,7 @@ dependencies {//api "com.kjetland:mbknor-jackson-jsonschema_2.12:1.0.39"api "org.scala-lang:scala-library:2.12.10"api "io.github.classgraph:classgraph:$classgraphVersion"+ api "io.micrometer:micrometer-core:$micrometerVersion"}jar {diff --git a/model/src/main/java/org/openremote/model/Container.java b/model/src/main/java/org/openremote/model/Container.javaindex 5a41d9e08..d8301d0ea 100644--- a/model/src/main/java/org/openremote/model/Container.java+++ b/model/src/main/java/org/openremote/model/Container.java@@ -19,6 +19,7 @@*/package org.openremote.model;+import io.micrometer.core.instrument.MeterRegistry;import org.openremote.model.util.TsIgnore;import java.util.Collection;@@ -30,6 +31,8 @@ public interface Container {String OR_DEV_MODE = "OR_DEV_MODE";boolean OR_DEV_MODE_DEFAULT = true;+ String OR_METRICS_ENABLED = "OR_METRICS_ENABLED";+ boolean OR_METRICS_ENABLED_DEFAULT = true;boolean isDevMode();@@ -44,4 +47,10 @@ public interface Container {<T extends ContainerService> T getService(Class<T> type) throws IllegalStateException;<T extends ContainerService> boolean hasService(Class<T> type);++ /**+ * Get the {@link MeterRegistry} to use for micrometer metrics gathering+ * @return Null if metrics has been disabled+ */+ MeterRegistry getMeterRegistry();}diff --git a/model/src/main/java/org/openremote/model/attribute/AttributeState.java b/model/src/main/java/org/openremote/model/attribute/AttributeState.javaindex 2339776f3..ae9be4b47 100644--- a/model/src/main/java/org/openremote/model/attribute/AttributeState.java+++ b/model/src/main/java/org/openremote/model/attribute/AttributeState.java@@ -79,9 +79,10 @@ public class AttributeState implements Serializable {@Overridepublic String toString() {+ String valueStr = Objects.toString(value);return getClass().getSimpleName() + "{" +"ref=" + ref +- ", value=" + value ++ ", value=" + (valueStr.length() > 100 ? valueStr.substring(0, 100) + "..." : valueStr) +", deleted=" + deleted +'}';}diff --git a/model/src/main/java/org/openremote/model/system/HealthStatusProvider.java b/model/src/main/java/org/openremote/model/system/HealthStatusProvider.javaindex f6a0fb95a..e77bdcfe4 100644--- a/model/src/main/java/org/openremote/model/system/HealthStatusProvider.java+++ b/model/src/main/java/org/openremote/model/system/HealthStatusProvider.java@@ -19,10 +19,12 @@*/package org.openremote.model.system;+import org.openremote.model.ContainerService;+/*** Allows implementors to provide health status data that is relevant to them.*/-public interface HealthStatusProvider {+public interface HealthStatusProvider extends ContainerService {String getHealthStatusName();diff --git a/model/src/main/java/org/openremote/model/system/StatusResource.java b/model/src/main/java/org/openremote/model/system/StatusResource.javaindex 45f8a1e9a..5971bf8c5 100644--- a/model/src/main/java/org/openremote/model/system/StatusResource.java+++ b/model/src/main/java/org/openremote/model/system/StatusResource.java@@ -34,6 +34,7 @@ import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON;@Path("")public interface StatusResource {+ // TODO: Replace this with prometheus metrics@Path("health")@GET@Produces(APPLICATION_JSON)diff --git a/profile/demo.yml b/profile/demo.ymlindex d06a9a486..a0fb03600 100644--- a/profile/demo.yml+++ b/profile/demo.yml@@ -46,6 +46,7 @@ services:- "80:80"- "443:443"- "8883:8883"+ - "127.0.0.1:8404:8404" # Localhost metrics accessvolumes:- proxy-data:/deploymentenvironment:@@ -84,6 +85,8 @@ services:depends_on:keycloak:condition: service_healthy+ ports:+ - "127.0.0.1:8405:8404" # Localhost metrics accessenvironment:# Set this variable `demo` to auto provision demo assets etc.OR_SETUP_TYPE: ${OR_SETUP_TYPE:-demo}diff --git a/profile/deploy.yml b/profile/deploy.ymlindex 3af13093a..8c25f21b1 100644--- a/profile/deploy.yml+++ b/profile/deploy.yml@@ -101,6 +101,7 @@ services:- "80:80" # HTTP for letsencrypt cert generation and in general it redirects to 443 for HTTPS- "${OR_SSL_PORT:-443}:443" # HTTPS /auth directs to keycloak and other paths direct to manager- "8883:8883" # MQTTS+ - "127.0.0.1:8404:8404" # Localhost metrics access# The OpenRemote Managermanager:@@ -125,6 +126,7 @@ services:# - "8080:8080" # Web server port# - "8000:8000" # Remote debugging port - NOT FOR PRODUCTION USE!!!# - "8085:8085" # JMX port - FOR DIAGNOSTICS ONLY!!!+ # - "127.0.0.1:8405:8404" # Localhost metrics accesslogging:options:# Use very little logging for container STDOUT, this is@@ -286,6 +288,11 @@ services:# processing push notifications) and will therefore fail silently until the user logs in again.OR_IDENTITY_SESSION_OFFLINE_TIMEOUT_MINUTES: ${OR_IDENTITY_SESSION_OFFLINE_TIMEOUT_MINUTES:-2628000}+ # Enable/disable Prometheus metrics collection+ OR_METRICS_ENABLED: ${OR_METRICS_ENABLED:-true}+ # Web server port used to host metrics endpoint+ OR_METRICS_PORT: ${OR_METRICS_PORT:-8404}+# Configure the default max age for data points stored in the database (individual attributes can use a custom# value by using the DATA_POINTS_MAX_AGE_DAYS AssetMeta item).# OR_DATA_POINTS_MAX_AGE_DAYS = 30@@ -312,8 +319,8 @@ services:# -Dcom.sun.management.jmxremote.rmi.port=8085# -Djava.rmi.server.hostname=${OR_HOSTNAME:-localhost}-# Custom JVM tool options (for remote debugging)-#JAVA_TOOL_OPTIONS: '-agentlib:jdwp=transport=dt_socket,address=8000,server=y,suspend=n'+ # Custom JVM tool options (for remote debugging)+ #JAVA_TOOL_OPTIONS: '-agentlib:jdwp=transport=dt_socket,address=0.0.0.0:8000,server=y,suspend=n'# Keycloak identity providerkeycloak:diff --git a/project.gradle b/project.gradleindex 16cc1e3cd..f5062f96b 100644--- a/project.gradle+++ b/project.gradle@@ -33,6 +33,10 @@ echo "***** Running gradle encryption plugin checkFilesGitIgnored task ******"status=\$?+if [ \$status != 0 ]; then+ echo "***** One or more encrypted files are not listed in a .gitignore - please add to prevent unencrypted version of file(s) from being committed *****"+fi+exit \$status"""Set<PosixFilePermission> perms = Files.readAttributes(path, PosixFileAttributes.class).permissions()diff --git a/setup/build.gradle b/setup/build.gradleindex 99b13c3d9..fc269c8de 100644--- a/setup/build.gradle+++ b/setup/build.gradle@@ -17,15 +17,22 @@ dependencies {implementation resolveProject(":manager")}-// Use demo sourceset for jar-jar {+task demoJar(type: Jar) {archivesBaseName = "openremote-demo-${project.name}"from sourceSets.demo.output}+task load1Jar(type: Jar) {+ archivesBaseName = "openremote-load1-${project.name}"+ from sourceSets.load1.output+}++// Use demo setup sourceset by default for jartask installDist(type: Copy) {- dependsOn jar+ def setupTask = getTasksByName(project.hasProperty("SETUP_JAR") ? "${project.getProperties().get("SETUP_JAR")}Jar" : "demoJar", false)[0] as org.gradle.jvm.tasks.Jar+ System.out.println(setupTask.name)+ dependsOn setupTaskdependsOn resolveTask(":manager:installDist")- from jar.archiveFile+ from setupTask.archiveFileinto "${resolveProject(":manager").buildDir}/install/manager/deployment/manager/extensions"}diff --git a/setup/src/load1/java/org/openremote/setup/load1/AttributeValueGeneratorService.java b/setup/src/load1/java/org/openremote/setup/load1/AttributeValueGeneratorService.javadeleted file mode 100644index 610748d33..000000000--- a/setup/src/load1/java/org/openremote/setup/load1/AttributeValueGeneratorService.java+++ /dev/null@@ -1,104 +0,0 @@-/*- * Copyright 2022, OpenRemote Inc.- *- * See the CONTRIBUTORS.txt file in the distribution for a- * full listing of individual contributors.- *- * This program is free software: you can redistribute it and/or modify- * it under the terms of the GNU Affero General Public License as- * published by the Free Software Foundation, either version 3 of the- * License, or (at your option) any later version.- *- * This program is distributed in the hope that it will be useful,- * but WITHOUT ANY WARRANTY; without even the implied warranty of- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the- * GNU Affero General Public License for more details.- *- * You should have received a copy of the GNU Affero General Public License- * along with this program. If not, see <http://www.gnu.org/licenses/>.- */-package org.openremote.setup.load1;--import org.apache.activemq.artemis.utils.collections.ConcurrentHashSet;-import org.apache.camel.builder.RouteBuilder;-import org.openremote.container.message.MessageBrokerService;-import org.openremote.manager.asset.AssetProcessingService;-import org.openremote.model.Container;-import org.openremote.model.ContainerService;-import org.openremote.model.PersistenceEvent;-import org.openremote.model.asset.Asset;-import org.openremote.model.asset.impl.WeatherAsset;-import org.openremote.model.attribute.AttributeEvent;--import java.util.Set;-import java.util.concurrent.ScheduledExecutorService;-import java.util.concurrent.ScheduledFuture;-import java.util.concurrent.TimeUnit;-import java.util.logging.Logger;--import static org.apache.camel.builder.PredicateBuilder.and;-import static org.openremote.container.persistence.PersistenceService.PERSISTENCE_TOPIC;-import static org.openremote.container.persistence.PersistenceService.isPersistenceEventForEntityType;--/**- * This service listens for {@link org.openremote.model.asset.impl.WeatherAsset}s to be added to the system then- * it generates attribute updates for the {@link WeatherAsset#RAINFALL} and {@link WeatherAsset#TEMPERATURE} attributes- * periodically- */-public class AttributeValueGeneratorService extends RouteBuilder implements ContainerService {-- protected final int UPDATE_MILLIS = 5000;- protected static final Logger LOG = Logger.getLogger(AttributeValueGeneratorService.class.getName());- protected AssetProcessingService assetProcessingService;- protected ScheduledExecutorService executorService;- protected Set<String> assetIds = new ConcurrentHashSet<>();- protected ScheduledFuture<?> updater;-- @Override- public void init(Container container) throws Exception {- assetProcessingService = container.getService(AssetProcessingService.class);- executorService = container.getExecutorService();- container.getService(MessageBrokerService.class).getContext().addRoutes(this);- }-- @Override- public void start(Container container) throws Exception {- updater = executorService.scheduleWithFixedDelay(this::sendValues, UPDATE_MILLIS, UPDATE_MILLIS, TimeUnit.MILLISECONDS);- }-- @Override- public void stop(Container container) throws Exception {- if (updater != null) {- updater.cancel(true);- }- updater = null;- }-- @SuppressWarnings("unchecked")- @Override- public void configure() throws Exception {- // If any asset was modified in the database, detect changed attributes- from(PERSISTENCE_TOPIC)- .routeId("WeatherAssetChanges")- .filter(and(isPersistenceEventForEntityType(Asset.class), exchange -> {- PersistenceEvent<Asset<?>> persistenceEvent = exchange.getIn().getBody(PersistenceEvent.class);- return persistenceEvent.getEntity() instanceof WeatherAsset && persistenceEvent.getCause() == PersistenceEvent.Cause.CREATE;- }))- .process(exchange -> {- PersistenceEvent<Asset<?>> persistenceEvent = exchange.getIn().getBody(PersistenceEvent.class);- onAssetCreated((WeatherAsset)persistenceEvent.getEntity());- });- }-- protected void onAssetCreated(WeatherAsset weatherAsset) {- assetIds.add(weatherAsset.getId());- }-- protected void sendValues() {- for (String assetId : assetIds) {- LOG.fine("Updating attributes on asset ID: " + assetId);- assetProcessingService.sendAttributeEvent(new AttributeEvent(assetId, WeatherAsset.TEMPERATURE, Math.random()*40));- assetProcessingService.sendAttributeEvent(new AttributeEvent(assetId, WeatherAsset.RAINFALL, Math.random()*10));- }- }-}diff --git a/setup/src/load1/java/org/openremote/setup/load1/ManagerSetup.java b/setup/src/load1/java/org/openremote/setup/load1/ManagerSetup.javaindex 2735fa82a..7a3769b8a 100644--- a/setup/src/load1/java/org/openremote/setup/load1/ManagerSetup.java+++ b/setup/src/load1/java/org/openremote/setup/load1/ManagerSetup.java@@ -69,12 +69,14 @@ public class ManagerSetup extends org.openremote.manager.setup.ManagerSetup {templateAsset.getAttribute(WeatherAsset.RAINFALL).ifPresent(attr -> attr.addMeta(new MetaItem<>(MetaItemType.ACCESS_RESTRICTED_READ),new MetaItem<>(MetaItemType.ACCESS_RESTRICTED_WRITE),- new MetaItem<>(MetaItemType.RULE_STATE)+ new MetaItem<>(MetaItemType.RULE_STATE),+ new MetaItem<>(MetaItemType.STORE_DATA_POINTS)));templateAsset.getAttribute(WeatherAsset.TEMPERATURE).ifPresent(attr -> attr.addMeta(new MetaItem<>(MetaItemType.ACCESS_RESTRICTED_READ),new MetaItem<>(MetaItemType.ACCESS_RESTRICTED_WRITE),- new MetaItem<>(MetaItemType.RULE_STATE)+ new MetaItem<>(MetaItemType.RULE_STATE),+ new MetaItem<>(MetaItemType.STORE_DATA_POINTS)));X509ProvisioningConfig provisioningConfig = new X509ProvisioningConfig("OEM Device",diff --git a/setup/src/load1/resources/META-INF/services/org.openremote.model.ContainerService b/setup/src/load1/resources/META-INF/services/org.openremote.model.ContainerServiceindex f0d4d9cf8..e69de29bb 100644--- a/setup/src/load1/resources/META-INF/services/org.openremote.model.ContainerService+++ b/setup/src/load1/resources/META-INF/services/org.openremote.model.ContainerService@@ -1 +0,0 @@-org.openremote.setup.load1.AttributeValueGeneratorServicediff --git a/test/build.gradle b/test/build.gradleindex 137d356ea..1d4039d01 100644--- a/test/build.gradle+++ b/test/build.gradle@@ -17,7 +17,7 @@ dependencies {testImplementation "org.slf4j:slf4j-api:$slf4jVersion"testImplementation resolveProject(":setup").sourceSets.integration.output- api resolveProject(":manager")+ testFixturesApi resolveProject(":manager")testFixturesImplementation "org.codehaus.groovy:groovy:$groovyVersion"testFixturesApi platform("org.spockframework:spock-bom:$spockVersion")diff --git a/test/load1/README.md b/test/load1/README.mdindex 8413801cb..5dd3389f2 100644--- a/test/load1/README.md+++ b/test/load1/README.md@@ -8,7 +8,10 @@ on changes to `rainfall` and `temperature` attributes* N x auto provisioning MQTT devices connecting to the MQTT broker and:* Subscribing to the auto provisioning response topic (1-N devices in M minutes)* Publishing their provisioning config requests (1-N devices in M minutes with initial delay of 10s)- * Publishing two attribute values (temperature and rainfall)+ * Subscribing to the attributes of the asset+ * Publishing two attribute values (temperature and rainfall) every R seconds+ * Checking for 3 attribute events to be received (temperature, rainfall and calculated)+ * Repeat the publish and checking indefinitely# Prerequisites@@ -20,7 +23,7 @@ on changes to `rainfall` and `temperature` attributes1. Run the `device_generator.sh COUNT` script to generate auto provisioning device X.509 certificates which are output to `tmp/devices.csv`the COUNT value is the number of devices to generate certificates for (default: 100)-3. Run the manager using the `load1` setup+3. Either run the manager using the `load1` setup or compile the `load1` setup jar (`./gradlew clean :setup:load1Jar`) manager with the `load1` setup included4. Run Taurus:```bzt mqtt_auto_provision.ymldiff --git a/test/load1/load1.jmx b/test/load1/load1.jmxnew file mode 100644index 000000000..81c86033b--- /dev/null+++ b/test/load1/load1.jmx@@ -0,0 +1,233 @@+<?xml version="1.0" encoding="UTF-8"?>+<jmeterTestPlan version="1.2" properties="5.0" jmeter="5.4.3">+ <hashTree>+ <TestPlan guiclass="TestPlanGui" testclass="TestPlan" testname="BZT Generated Test Plan" enabled="true">+ <elementProp name="TestPlan.user_defined_variables" elementType="Arguments" guiclass="ArgumentsPanel" testclass="Arguments" enabled="true">+ <collectionProp name="Arguments.arguments"/>+ </elementProp>+ <boolProp name="TestPlan.functional_mode">false</boolProp>+ <boolProp name="TestPlan.serialize_threadgroups">false</boolProp>+ <stringProp name="TestPlan.comments"></stringProp>+ <stringProp name="TestPlan.user_define_classpath"></stringProp>+ </TestPlan>+ <hashTree>+ <ThreadGroup guiclass="ThreadGroupGui" testclass="ThreadGroup" testname="mqtt-client-auto-provision-pub" enabled="true">+ <stringProp name="ThreadGroup.on_sample_error">continue</stringProp>+ <elementProp name="ThreadGroup.main_controller" elementType="LoopController" guiclass="LoopControlPanel" testclass="LoopController" enabled="true">+ <boolProp name="LoopController.continue_forever">false</boolProp>+ <stringProp name="LoopController.loops">1</stringProp>+ </elementProp>+ <stringProp name="ThreadGroup.num_threads">1</stringProp>+ <stringProp name="ThreadGroup.ramp_time">1</stringProp>+ <stringProp name="ThreadGroup.start_time"></stringProp>+ <stringProp name="ThreadGroup.end_time"></stringProp>+ <boolProp name="ThreadGroup.scheduler">false</boolProp>+ <stringProp name="ThreadGroup.duration">0</stringProp>+ <stringProp name="ThreadGroup.delay"></stringProp>+ <boolProp name="ThreadGroup.same_user_on_next_iteration">true</boolProp>+ </ThreadGroup>+ <hashTree>+ <CacheManager guiclass="CacheManagerGui" testclass="CacheManager" testname="Cache" enabled="true">+ <boolProp name="clearEachIteration">true</boolProp>+ <boolProp name="useExpires">true</boolProp>+ <boolProp name="CacheManager.controlledByThread">false</boolProp>+ </CacheManager>+ <hashTree/>+ <CookieManager guiclass="CookiePanel" testclass="CookieManager" testname="Cookies" enabled="true">+ <collectionProp name="CookieManager.cookies"/>+ <boolProp name="CookieManager.clearEachIteration">false</boolProp>+ <boolProp name="CookieManager.controlledByThreadGroup">false</boolProp>+ </CookieManager>+ <hashTree/>+ <DNSCacheManager guiclass="DNSCachePanel" testclass="DNSCacheManager" testname="DNS Cache Manager" enabled="true">+ <collectionProp name="DNSCacheManager.servers"/>+ <boolProp name="DNSCacheManager.clearEachIteration">false</boolProp>+ <boolProp name="DNSCacheManager.isCustomResolver">false</boolProp>+ </DNSCacheManager>+ <hashTree/>+ <ConfigTestElement guiclass="HttpDefaultsGui" testclass="ConfigTestElement" testname="Defaults" enabled="true">+ <elementProp name="HTTPsampler.Arguments" elementType="Arguments" guiclass="HTTPArgumentsPanel" testclass="Arguments" testname="user_defined" enabled="true">+ <collectionProp name="Arguments.arguments"/>+ </elementProp>+ <stringProp name="HTTPSampler.domain"></stringProp>+ <stringProp name="HTTPSampler.port"></stringProp>+ <stringProp name="HTTPSampler.protocol"></stringProp>+ <stringProp name="HTTPSampler.contentEncoding"></stringProp>+ <stringProp name="HTTPSampler.path"></stringProp>+ <boolProp name="HTTPSampler.image_parser">true</boolProp>+ <boolProp name="HTTPSampler.concurrentDwn">true</boolProp>+ <stringProp name="HTTPSampler.concurrentPool">4</stringProp>+ <stringProp name="HTTPSampler.connect_timeout">30000</stringProp>+ <stringProp name="HTTPSampler.response_timeout">30000</stringProp>+ </ConfigTestElement>+ <hashTree/>+ <CSVDataSet guiclass="TestBeanGUI" testclass="CSVDataSet" testname="CSV devices.csv" enabled="true">+ <stringProp name="filename">C:\Users\Richard\code\openremote\test\load1\tmp\devices.csv</stringProp>+ <stringProp name="delimiter">,</stringProp>+ <boolProp name="quotedData">false</boolProp>+ <boolProp name="recycle">false</boolProp>+ <boolProp name="stopThread">true</boolProp>+ <stringProp name="variableNames">id,cert</stringProp>+ <stringProp name="fileEncoding"></stringProp>+ <boolProp name="ignoreFirstLine">false</boolProp>+ <stringProp name="shareMode">shareMode.all</stringProp>+ </CSVDataSet>+ <hashTree/>+ <net.xmeter.samplers.ConnectSampler guiclass="net.xmeter.gui.ConnectSamplerUI" testclass="net.xmeter.samplers.ConnectSampler" testname="connect" enabled="true">+ <stringProp name="mqtt.server">localhost</stringProp>+ <stringProp name="mqtt.port">1883</stringProp>+ <stringProp name="mqtt.version">3.1</stringProp>+ <stringProp name="mqtt.conn_timeout">10</stringProp>+ <stringProp name="mqtt.protocol">TCP</stringProp>+ <stringProp name="mqtt.ws_path"></stringProp>+ <boolProp name="mqtt.dual_ssl_authentication">false</boolProp>+ <stringProp name="mqtt.clientcert_file_path"></stringProp>+ <stringProp name="mqtt.clientcert_password"></stringProp>+ <stringProp name="mqtt.client_id_prefix">${id}</stringProp>+ <boolProp name="mqtt.client_id_suffix">false</boolProp>+ <stringProp name="mqtt.conn_keep_alive">300</stringProp>+ <stringProp name="mqtt.conn_attampt_max">0</stringProp>+ <stringProp name="mqtt.reconn_attampt_max">0</stringProp>+ <stringProp name="mqtt.conn_clean_session">true</stringProp>+ <stringProp name="mqtt.user_name"></stringProp>+ <stringProp name="mqtt.password"></stringProp>+ <stringProp name="mqtt.connect_name">mqttconn</stringProp>+ <stringProp name="mqtt.conn_attempt_max">0</stringProp>+ <stringProp name="mqtt.reconn_attempt_max">0</stringProp>+ </net.xmeter.samplers.ConnectSampler>+ <hashTree/>+ <net.xmeter.samplers.SubSampler guiclass="net.xmeter.gui.SubSamplerUI" testclass="net.xmeter.samplers.SubSampler" testname="Subscribe provisioning response" enabled="true">+ <stringProp name="mqtt.topic_name">provisioning/${id}/response</stringProp>+ <stringProp name="mqtt.qos_level">0</stringProp>+ <boolProp name="mqtt.add_timestamp">false</boolProp>+ <boolProp name="mqtt.debug_response">true</boolProp>+ <boolProp name="mqtt.sub_async">true</boolProp>+ <stringProp name="mqtt.sample_condition">number of received messages</stringProp>+ <stringProp name="mqtt.sample_condition_value">1</stringProp>+ <stringProp name="mqtt.connect_name">mqttconn</stringProp>+ <stringProp name="mqtt.sample_condition_value_opt">10000</stringProp>+ </net.xmeter.samplers.SubSampler>+ <hashTree/>+ <net.xmeter.samplers.PubSampler guiclass="net.xmeter.gui.PubSamplerUI" testclass="net.xmeter.samplers.PubSampler" testname="Publish provisioning" enabled="true">+ <stringProp name="mqtt.topic_name">provisioning/${id}/request</stringProp>+ <stringProp name="mqtt.qos_level">0</stringProp>+ <boolProp name="mqtt.add_timestamp">false</boolProp>+ <stringProp name="mqtt.message_type">String</stringProp>+ <stringProp name="mqtt.message_type_fixed_length">1024</stringProp>+ <stringProp name="mqtt.message_to_sent">{"type":"x509","cert":"${cert}"}</stringProp>+ <boolProp name="mqtt.retained_message">false</boolProp>+ <stringProp name="mqtt.connect_name">mqttconn</stringProp>+ </net.xmeter.samplers.PubSampler>+ <hashTree/>+ <net.xmeter.samplers.AsyncResponseSampler guiclass="net.xmeter.gui.AsyncResponseSamplerUI" testclass="net.xmeter.samplers.AsyncResponseSampler" testname="Check provisioning response" enabled="true"/>+ <hashTree>+ <RegexExtractor guiclass="RegexExtractorGui" testclass="RegexExtractor" testname="Regular Expression Extractor" enabled="true">+ <stringProp name="RegexExtractor.useHeaders">false</stringProp>+ <stringProp name="RegexExtractor.refname">assetId</stringProp>+ <stringProp name="RegexExtractor.regex">"id"\s*:\s*"(\w{22})"</stringProp>+ <stringProp name="RegexExtractor.template">$1$</stringProp>+ <stringProp name="RegexExtractor.default"></stringProp>+ <stringProp name="RegexExtractor.match_number">1</stringProp>+ </RegexExtractor>+ <hashTree/>+ </hashTree>+ <net.xmeter.samplers.SubSampler guiclass="net.xmeter.gui.SubSamplerUI" testclass="net.xmeter.samplers.SubSampler" testname="Subscribe attributes" enabled="true">+ <stringProp name="mqtt.topic_name">master/${id}/attributevalue/+/${assetId}</stringProp>+ <stringProp name="mqtt.qos_level">0</stringProp>+ <boolProp name="mqtt.add_timestamp">false</boolProp>+ <boolProp name="mqtt.debug_response">true</boolProp>+ <stringProp name="mqtt.sample_condition">number of received messages</stringProp>+ <stringProp name="mqtt.sample_condition_value">3</stringProp>+ <boolProp name="mqtt.sub_async">true</boolProp>+ <stringProp name="mqtt.connect_name">mqttconn</stringProp>+ <stringProp name="mqtt.sample_condition_value_opt">10000</stringProp>+ </net.xmeter.samplers.SubSampler>+ <hashTree/>+ <LoopController guiclass="LoopControlPanel" testclass="LoopController" testname="Loop Controller" enabled="true">+ <boolProp name="LoopController.continue_forever">true</boolProp>+ <intProp name="LoopController.loops">-1</intProp>+ </LoopController>+ <hashTree>+ <net.xmeter.samplers.PubSampler guiclass="net.xmeter.gui.PubSamplerUI" testclass="net.xmeter.samplers.PubSampler" testname="Publish rainfall" enabled="true">+ <stringProp name="mqtt.topic_name">master/${id}/writeattributevalue/rainfall/${assetId}</stringProp>+ <stringProp name="mqtt.qos_level">0</stringProp>+ <boolProp name="mqtt.add_timestamp">false</boolProp>+ <stringProp name="mqtt.message_type">String</stringProp>+ <stringProp name="mqtt.message_type_fixed_length">1024</stringProp>+ <stringProp name="mqtt.message_to_sent">${__Random(0,10)}</stringProp>+ <boolProp name="mqtt.retained_message">false</boolProp>+ <stringProp name="mqtt.connect_name">mqttconn</stringProp>+ </net.xmeter.samplers.PubSampler>+ <hashTree/>+ <net.xmeter.samplers.PubSampler guiclass="net.xmeter.gui.PubSamplerUI" testclass="net.xmeter.samplers.PubSampler" testname="Publish temperature" enabled="true">+ <stringProp name="mqtt.topic_name">master/${id}/writeattributevalue/temperature/${assetId}</stringProp>+ <stringProp name="mqtt.qos_level">0</stringProp>+ <boolProp name="mqtt.add_timestamp">false</boolProp>+ <stringProp name="mqtt.message_type">String</stringProp>+ <stringProp name="mqtt.message_type_fixed_length">1024</stringProp>+ <stringProp name="mqtt.message_to_sent">${__Random(10,20)}</stringProp>+ <boolProp name="mqtt.retained_message">false</boolProp>+ <stringProp name="mqtt.connect_name">mqttconn</stringProp>+ </net.xmeter.samplers.PubSampler>+ <hashTree/>+ <net.xmeter.samplers.AsyncResponseSampler guiclass="net.xmeter.gui.AsyncResponseSamplerUI" testclass="net.xmeter.samplers.AsyncResponseSampler" testname="Check attribute response" enabled="true"/>+ <hashTree>+ <ConstantTimer guiclass="ConstantTimerGui" testclass="ConstantTimer" testname="Constant Timer" enabled="false">+ <stringProp name="ConstantTimer.delay">50</stringProp>+ </ConstantTimer>+ <hashTree/>+ </hashTree>+ </hashTree>+ <JSR223PostProcessor guiclass="TestBeanGUI" testclass="JSR223PostProcessor" testname="JSR223PostProcessor" enabled="false">+ <stringProp name="filename">C:\Users\Richard\code\openremote\test\load1\output\2022-11-24_17-00-55.939257\mqtt-subscribe.gsh</stringProp>+ <stringProp name="script"></stringProp>+ <stringProp name="parameters"></stringProp>+ <stringProp name="scriptLanguage">groovy</stringProp>+ <stringProp name="cacheKey">true</stringProp>+ </JSR223PostProcessor>+ <hashTree/>+ <net.xmeter.samplers.DisConnectSampler guiclass="net.xmeter.gui.DisConnectSamplerUI" testclass="net.xmeter.samplers.DisConnectSampler" testname="disconnect" enabled="true">+ <stringProp name="mqtt.connect_name">mqttconn</stringProp>+ </net.xmeter.samplers.DisConnectSampler>+ <hashTree/>+ <ResultCollector guiclass="ViewResultsFullVisualizer" testclass="ResultCollector" testname="View Results Tree" enabled="true">+ <boolProp name="ResultCollector.error_logging">false</boolProp>+ <objProp>+ <name>saveConfig</name>+ <value class="SampleSaveConfiguration">+ <time>true</time>+ <latency>true</latency>+ <timestamp>true</timestamp>+ <success>true</success>+ <label>true</label>+ <code>true</code>+ <message>true</message>+ <threadName>true</threadName>+ <dataType>true</dataType>+ <encoding>false</encoding>+ <assertions>true</assertions>+ <subresults>true</subresults>+ <responseData>false</responseData>+ <samplerData>false</samplerData>+ <xml>false</xml>+ <fieldNames>true</fieldNames>+ <responseHeaders>false</responseHeaders>+ <requestHeaders>false</requestHeaders>+ <responseDataOnError>false</responseDataOnError>+ <saveAssertionResultsFailureMessage>true</saveAssertionResultsFailureMessage>+ <assertionsResultsToSave>0</assertionsResultsToSave>+ <bytes>true</bytes>+ <sentBytes>true</sentBytes>+ <url>true</url>+ <threadCounts>true</threadCounts>+ <idleTime>true</idleTime>+ <connectTime>true</connectTime>+ </value>+ </objProp>+ <stringProp name="filename"></stringProp>+ </ResultCollector>+ <hashTree/>+ </hashTree>+ </hashTree>+ </hashTree>+</jmeterTestPlan>diff --git a/test/src/logging-test.properties b/test/src/logging-test.propertiesindex f7fef93fd..2e8dbd251 100644--- a/test/src/logging-test.properties+++ b/test/src/logging-test.properties@@ -31,7 +31,6 @@ java.util.logging.ConsoleHandler.formatter=org.openremote.container.util.LogFormorg.openremote.level=FINE# Processing time logging-#org.openremote.container.concurrent.GlobalLock.level=FINE#org.openremote.manager.asset.AssetProcessingService.level=FINE#org.openremote.manager.asset.AssetStorageService.level=FINE#org.openremote.manager.rules.level=FINEdiff --git a/test/src/test/groovy/org/openremote/test/assets/AssetProcessingTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetProcessingTest.groovyindex c70f63bc6..96969650b 100644--- a/test/src/test/groovy/org/openremote/test/assets/AssetProcessingTest.groovy+++ b/test/src/test/groovy/org/openremote/test/assets/AssetProcessingTest.groovy@@ -123,7 +123,7 @@ class AssetProcessingTest extends Specification implements ManagerContainerTraitthen: "all agents should become connected"conditions.eventually {- assert agentService.agentMap.values().every {it.getAgentStatus().orElse(null) == ConnectionStatus.CONNECTED}+ assert agentService.agents.values().every {it.getAgentStatus().orElse(null) == ConnectionStatus.CONNECTED}}then: "the mock thing to be deployed to the protocol instance"diff --git a/test/src/test/groovy/org/openremote/test/gateway/GatewayTest.groovy b/test/src/test/groovy/org/openremote/test/gateway/GatewayTest.groovyindex 3698ee615..5fb8f98cb 100644--- a/test/src/test/groovy/org/openremote/test/gateway/GatewayTest.groovy+++ b/test/src/test/groovy/org/openremote/test/gateway/GatewayTest.groovy@@ -340,9 +340,9 @@ class GatewayTest extends Specification implements ManagerContainerTrait {Thread.sleep(500)conditions.eventually {assert !agentService.protocolInstanceMap.containsKey(agentAssetIds[0])- assert !agentService.agentMap.containsKey(agentAssetIds[0])+ assert !agentService.agents.containsKey(agentAssetIds[0])assert !agentService.protocolInstanceMap.containsKey(agentAssetIds[4])- assert !agentService.agentMap.containsKey(agentAssetIds[4])+ assert !agentService.agents.containsKey(agentAssetIds[4])}when: "the previously received messages are cleared"diff --git a/test/src/test/groovy/org/openremote/test/mqtt/MqttBrokerTest.groovy b/test/src/test/groovy/org/openremote/test/mqtt/MqttBrokerTest.groovyindex 9e4a6c0d9..3031bb40c 100644--- a/test/src/test/groovy/org/openremote/test/mqtt/MqttBrokerTest.groovy+++ b/test/src/test/groovy/org/openremote/test/mqtt/MqttBrokerTest.groovy@@ -615,19 +615,49 @@ class MqttBrokerTest extends Specification implements ManagerContainerTrait {assert clientEventService.eventSubscriptions.sessionSubscriptionIdMap.get(getConnectionIDString(connection)).size() == 1}- when: "user asset links are updated for a connected restricted user"+ when: "a user asset link is added for a connected restricted user"existingConnection = mqttBrokerService.getUserConnections(keycloakTestSetup.serviceUser2.id)[0]assetStorageService.storeUserAssetLinks(List.of(new UserAssetLink(keycloakTestSetup.realmBuilding.getName(),keycloakTestSetup.serviceUser2.getId(),managerTestSetup.apartment1BathroomId)))+ then: "the existing connection should not have been terminated"+ new PollingConditions(initialDelay: 2, timeout: 10, delay: 1).eventually {+ assert mqttBrokerService.getUserConnections(keycloakTestSetup.serviceUser2.id).size() == 1+ assert mqttBrokerService.getUserConnections(keycloakTestSetup.serviceUser2.id)[0].is(existingConnection)+ }++ when: "a user asset link is removed for a connected restricted user"+ existingConnection = mqttBrokerService.getUserConnections(keycloakTestSetup.serviceUser2.id)[0]+ assetStorageService.deleteUserAssetLinks(List.of(+ new UserAssetLink(keycloakTestSetup.realmBuilding.getName(),+ keycloakTestSetup.serviceUser2.getId(),+ managerTestSetup.apartment1HallwayId)))+then: "the existing connection should be terminated and the client should reconnect"conditions.eventually {assert mqttBrokerService.getUserConnections(keycloakTestSetup.serviceUser2.id).size() == 1assert mqttBrokerService.getUserConnections(keycloakTestSetup.serviceUser2.id)[0] !== existingConnection}+ when: "the restricted mqtt client removes all subscriptions"+ newClient.removeAllMessageConsumers() // Clear subscriptions as otherwise it won't hit the server again++ then: "no subscriptions should exist in the client"+ assert newClient.topicConsumerMap.isEmpty()++ when: "the restricted mqtt client subscribes to a now unlinked asset"+ topic = "${keycloakTestSetup.realmBuilding.name}/$newClientId/$DefaultMQTTHandler.ATTRIBUTE_TOPIC/$MQTTHandler.TOKEN_SINGLE_LEVEL_WILDCARD/$managerTestSetup.apartment1HallwayId".toString()+ newClient.addMessageConsumer(topic, {msg ->})++ then: "no subscription should exist"+ conditions.eventually {+ assert newClient.topicConsumerMap.get(topic) == null // Consumer added and removed on failure+ def connection = mqttBrokerService.getUserConnections(keycloakTestSetup.serviceUser2.id)[0]+ assert !clientEventService.eventSubscriptions.sessionSubscriptionIdMap.containsKey(getConnectionIDString(connection))+ }+// TODO: Further MQTT tests// when: "an attribute event occurs on an attribute that the restricted user can access"//diff --git a/test/src/test/groovy/org/openremote/test/protocol/BasicProtocolTest.groovy b/test/src/test/groovy/org/openremote/test/protocol/BasicProtocolTest.groovyindex 5ee4a7718..f57ca1d83 100644--- a/test/src/test/groovy/org/openremote/test/protocol/BasicProtocolTest.groovy+++ b/test/src/test/groovy/org/openremote/test/protocol/BasicProtocolTest.groovy@@ -81,7 +81,7 @@ class BasicProtocolTest extends Specification implements ManagerContainerTrait {then: "the protocol instances should have been created and the agent status attributes should be updated"conditions.eventually {- assert agentService.agentMap.values().count {it instanceof MockAgent} == 3+ assert agentService.agents.values().count {it instanceof MockAgent} == 3assert agentService.protocolInstanceMap.values().count {it instanceof MockProtocol} == 1assert agentService.getAgent(mockAgent1.id) != nullassert agentService.getAgent(mockAgent2.id) != nulldiff --git a/test/src/test/groovy/org/openremote/test/protocol/mqtt/MQTTClientProtocolTest.groovy b/test/src/test/groovy/org/openremote/test/protocol/mqtt/MQTTClientProtocolTest.groovyindex d23b1ae52..97005535e 100644--- a/test/src/test/groovy/org/openremote/test/protocol/mqtt/MQTTClientProtocolTest.groovy+++ b/test/src/test/groovy/org/openremote/test/protocol/mqtt/MQTTClientProtocolTest.groovy@@ -46,6 +46,7 @@ import org.openremote.setup.integration.ManagerTestSetupimport spock.lang.Specificationimport spock.util.concurrent.PollingConditions+import static org.openremote.manager.mqtt.MQTTBrokerService.getConnectionIDStringimport static org.openremote.model.value.MetaItemType.AGENT_LINKimport static org.openremote.model.value.ValueType.NUMBER@@ -107,6 +108,9 @@ class MQTTClientProtocolTest extends Specification implements ManagerContainerTrthen: "the linked attributes should be correctly linked"conditions.eventually {assert !(agentService.getProtocolInstance(agent.id) as MQTTProtocol).protocolMessageConsumers.isEmpty()+ def connection = brokerService.getConnectionFromClientID(clientId)+ assert connection != null+ assert clientEventService.eventSubscriptions.sessionSubscriptionIdMap.containsKey(getConnectionIDString(connection))}when: "the attribute referenced in the agent link is updated"diff --git a/test/src/test/groovy/org/openremote/test/protocol/udp/UdpClientProtocolTest.groovy b/test/src/test/groovy/org/openremote/test/protocol/udp/UdpClientProtocolTest.groovyindex b70cb4346..3b623c6a2 100644--- a/test/src/test/groovy/org/openremote/test/protocol/udp/UdpClientProtocolTest.groovy+++ b/test/src/test/groovy/org/openremote/test/protocol/udp/UdpClientProtocolTest.groovy@@ -107,7 +107,7 @@ class UdpClientProtocolTest extends Specification implements ManagerContainerTrathen: "the protocol instance should be created and should become connected"conditions.eventually {assert agentService.getProtocolInstance(agent.id) != null- assert agentService.agentMap.get(agent.id).getAgentStatus().orElse(null) == ConnectionStatus.CONNECTED+ assert agentService.agents.get(agent.id).getAgentStatus().orElse(null) == ConnectionStatus.CONNECTED}when: "an asset is created with attributes linked to the agent"@@ -283,8 +283,8 @@ class UdpClientProtocolTest extends Specification implements ManagerContainerTrathen: "the protocol should be relinked"conditions.eventually {- assert agentService.agentMap.get(agent.id) != null- assert ((UDPAgent)agentService.agentMap.get(agent.id)).getMessageConvertHex().orElse(false)+ assert agentService.agents.get(agent.id) != null+ assert ((UDPAgent)agentService.agents.get(agent.id)).getMessageConvertHex().orElse(false)assert agentService.getProtocolInstance(agent.id) != nullassert agentService.getProtocolInstance(agent.id).linkedAttributes.size() == 7assert ((UDPProtocol)agentService.getProtocolInstance(agent.id)).protocolMessageConsumers.size() == 3diff --git a/test/src/test/groovy/org/openremote/test/rules/BasicRulesetResourceTest.groovy b/test/src/test/groovy/org/openremote/test/rules/BasicRulesetResourceTest.groovyindex 2e66cb488..46d2ee07e 100644--- a/test/src/test/groovy/org/openremote/test/rules/BasicRulesetResourceTest.groovy+++ b/test/src/test/groovy/org/openremote/test/rules/BasicRulesetResourceTest.groovy@@ -142,6 +142,7 @@ class BasicRulesetResourceTest extends Specification implements ManagerContainerand: "the ruleset should be removed from the engine"conditions.eventually {+ assert rulesService.globalEngine != nulldef deployment = rulesService.globalEngine.deployments.get(rulesetId)assert deployment == null}diff --git a/test/src/test/resources/org/openremote/test/rules/ChildAssetControl.groovy b/test/src/test/resources/org/openremote/test/rules/ChildAssetControl.groovyindex 1bb66238e..a16f74b73 100644--- a/test/src/test/resources/org/openremote/test/rules/ChildAssetControl.groovy+++ b/test/src/test/resources/org/openremote/test/rules/ChildAssetControl.groovy@@ -29,7 +29,7 @@ rules.add().when({ facts ->- List<AssetState<?>> changes = []+ List<AssetState<?>> changes = Collections.synchronizedList(new ArrayList<>())def childUpdates = facts.matchAssetState(new AssetQuery().ids(parentAssetId)diff --git a/test/src/testFixtures/groovy/org/openremote/test/ContainerTrait.groovy b/test/src/testFixtures/groovy/org/openremote/test/ContainerTrait.groovyindex eacf4c13d..2c42538f9 100644--- a/test/src/testFixtures/groovy/org/openremote/test/ContainerTrait.groovy+++ b/test/src/testFixtures/groovy/org/openremote/test/ContainerTrait.groovy@@ -19,8 +19,9 @@*/package org.openremote.test+import jakarta.persistence.TypedQuery+import jakarta.ws.rs.core.UriBuilderimport org.apache.camel.ProducerTemplate-import org.apache.camel.builder.RouteBuilderimport org.jboss.resteasy.client.jaxrs.ResteasyClientimport org.jboss.resteasy.client.jaxrs.ResteasyClientBuilderimport org.jboss.resteasy.client.jaxrs.ResteasyWebTarget@@ -40,32 +41,27 @@ import org.openremote.manager.agent.AgentServiceimport org.openremote.manager.asset.AssetProcessingServiceimport org.openremote.manager.asset.AssetStorageServiceimport org.openremote.manager.gateway.GatewayClientService+import org.openremote.manager.rules.RulesEngineimport org.openremote.manager.rules.RulesServiceimport org.openremote.manager.rules.RulesetStorageServiceimport org.openremote.manager.security.ManagerIdentityServiceimport org.openremote.manager.web.ManagerWebService-import org.openremote.model.Constantsimport org.openremote.model.ContainerServiceimport org.openremote.model.asset.Assetimport org.openremote.model.asset.UserAssetLinkimport org.openremote.model.asset.agent.Agent+import org.openremote.model.asset.agent.Protocolimport org.openremote.model.gateway.GatewayConnectionimport org.openremote.model.query.AssetQueryimport org.openremote.model.query.RulesetQueryimport org.openremote.model.rules.AssetRulesetimport org.openremote.model.rules.GlobalRuleset-import org.openremote.model.rules.Rulesetimport org.openremote.model.rules.RealmRuleset+import org.openremote.model.rules.Rulesetimport org.openremote.model.security.Userimport org.slf4j.Loggerimport org.slf4j.LoggerFactory-import jakarta.persistence.TypedQuery-import jakarta.websocket.ClientEndpointConfig-import jakarta.websocket.Endpoint-import jakarta.websocket.Session-import jakarta.websocket.WebSocketContainer-import jakarta.ws.rs.core.UriBuilderimport java.util.concurrent.TimeUnitimport java.util.logging.Handlerimport java.util.stream.Collectors@@ -121,6 +117,7 @@ trait ContainerTrait {if (configsMatch && servicesMatch) {try {+ long startTime = System.currentTimeMillis()LOG.info("Services and config matches already running container so checking state")def counter = 0@@ -152,67 +149,58 @@ trait ContainerTrait {// Reset rulesetsif (container.hasService(RulesetStorageService.class) && container.hasService(RulesService.class)) {- def globalRulesets = getRulesets(GlobalRuleset.class)- def realmRulesets = getRulesets(RealmRuleset.class)- def assetRulesets = getRulesets(AssetRuleset.class)def rulesService = container.getService(RulesService.class)def rulesetStorageService = container.getService(RulesetStorageService.class)- if (!assetRulesets.isEmpty()) {- LOG.info("Purging ${assetRulesets.size()} asset ruleset(s)")- assetRulesets.forEach {- rulesetStorageService.delete(AssetRuleset.class, it.id)- def assetEngine = rulesService.assetEngines.get(((AssetRuleset) it).assetId)- def rulesStopped = false- while (assetEngine != null && !rulesStopped) {- rulesStopped = assetEngine.deployments.containsKey(it.id)- if (counter++ > 100) {- throw new IllegalStateException("Failed to purge ruleset: " + it.name)- }- Thread.sleep(100)+ def assetEngines = new HashSet<RulesEngine>(rulesService.assetEngines.values())+ LOG.info("Purging ${assetEngines.size()} asset engine(s)")+ assetEngines.forEach {+ it.stop(true)+ while (!it.deployments.isEmpty()) {+ if (counter++ > 20) {+ throw new IllegalStateException("Failed to stop asset engine: " + it.id)}+ Thread.sleep(100)}+ rulesService.assetEngines.values().remove(it)}- if (!realmRulesets.isEmpty()) {- LOG.info("Purging ${realmRulesets.size()} realm ruleset(s)")- realmRulesets.forEach {- rulesetStorageService.delete(RealmRuleset.class, it.id)- def realmEngine = rulesService.realmEngines.get(((RealmRuleset) it).realm)- def rulesStopped = false- while (realmEngine != null && !rulesStopped) {- rulesStopped = !realmEngine.deployments.containsKey(it.id)- if (counter++ > 100) {- throw new IllegalStateException("Failed to purge ruleset: " + it.name)- }- Thread.sleep(100)- }- }+ def assetRulesets = getRulesets(AssetRuleset.class)+ assetRulesets.forEach{+ rulesetStorageService.delete(AssetRuleset.class, it.id)}- if (!globalRulesets.isEmpty()) {- LOG.info("Purging ${globalRulesets.size()} global ruleset(s)")- globalRulesets.forEach {- rulesetStorageService.delete(GlobalRuleset.class, it.id)- def rulesStopped = false- while (!rulesStopped) {- def engine = rulesService.globalEngine- rulesStopped = engine == null || !engine.deployments.containsKey(it.id)- if (counter++ > 100) {- throw new IllegalStateException("Failed to purge ruleset: " + it.name)- }- Thread.sleep(100)++ def realmEngines = new HashSet<RulesEngine>(rulesService.realmEngines.values())+ LOG.info("Purging ${realmEngines.size()} realm engine(s)")+ realmEngines.forEach {+ it.stop(true)+ while (!it.deployments.isEmpty()) {+ if (counter++ > 20) {+ throw new IllegalStateException("Failed to stop realm engine: " + it.id)}+ Thread.sleep(100)}+ rulesService.realmEngines.values().remove(it)+ }+ def realmRulesets = getRulesets(RealmRuleset.class)+ realmRulesets.forEach{+ rulesetStorageService.delete(RealmRuleset.class, it.id)}- // Wait for all rule engines to stop and be removed- counter = 0- def enginesStopped = false- while (!enginesStopped) {- enginesStopped = (rulesService.realmEngines == null || rulesService.realmEngines.isEmpty()) && (rulesService.assetEngines == null || rulesService.assetEngines.isEmpty())- if (counter++ > 100) {- throw new IllegalStateException("Rule engines have failed to stop")+ def globalEngine = rulesService.globalEngine+ if (globalEngine != null) {+ LOG.info("Purging global engine")+ globalEngine.stop(true)+ while (!globalEngine.deployments.isEmpty()) {+ if (counter++ > 20) {+ throw new IllegalStateException("Failed to stop global engine: " + globalEngine.id)+ }+ Thread.sleep(100)}- Thread.sleep(100)+ rulesService.globalEngine = null+ }+ def globalRulesets = getRulesets(GlobalRuleset.class)+ globalRulesets.forEach{+ rulesetStorageService.delete(GlobalRuleset.class, it.id)}}@@ -243,7 +231,7 @@ trait ContainerTrait {if (counter++ > 100) {throw new IllegalStateException("Failed to purge agents")}- agentsRemoved = container.getService(AgentService.class).agentMap.isEmpty()+ agentsRemoved = container.getService(AgentService.class).agents.isEmpty()Thread.sleep(100)}@@ -279,8 +267,11 @@ trait ContainerTrait {TestFixture.globalRulesets.forEach { rulesetStorageService.merge(it) }}}++ long endTime = System.currentTimeMillis()+ LOG.info("Container reuse took: " + (startTime - endTime) + "ms")} catch (IllegalStateException e) {- LOG.info("Failed to clean the existing container so creating a new one")+ LOG.info("Failed to clean the existing container so creating a new one", e)stopContainer()TestFixture.container = null}@@ -292,8 +283,14 @@ trait ContainerTrait {}if (container == null) {- TestFixture.container = new Container(config, services)- container.startBackground()+ try {+ TestFixture.container = new Container(config, services)+ container.startBackground()+ } catch (Exception e) {+ LOG.warn("Failed to start the container")+ stopContainer()+ throw e+ }// Block until container is actually running to aid in testing but also to record some state infowhile (!container.isRunning()) {@@ -325,11 +322,27 @@ trait ContainerTrait {if (agentService != null) {LOG.info("Waiting for agents to be deployed")i=0- while (i < 100 && TestFixture.assets.stream().filter { it instanceof Agent }.any { !agentService.agentMap.containsKey(it.id) }) {+ while (i < 100 && TestFixture.assets.stream().filter { it instanceof Agent }.any {+ def agent = agentService.agents.get(it.id)+ if (agent == null) {+ return true+ }+ Protocol<?> protocolInstance = agentService.protocolInstanceMap.get(it.id)+ if (protocolInstance == null) {+ return true+ }+ return !protocolInstance.agent.is(agent)+ }) {Thread.sleep(100)i++}- LOG.info("Agents are deployed")+ if (i >= 100) {+ LOG.info("Agents didn't load correctly so stopping and starting the container")+ stopContainer()+ startContainer(config, services)+ } else {+ LOG.info("Agents are deployed")+ }}if (rulesService != null) {@@ -473,8 +486,12 @@ trait ContainerTrait {}void stopContainer() {- if (container != null) {- container.stop()+ try {+ if (container != null) {+ container.stop()+ }+ } catch (Exception e) {+ LOG.warn("Failed to stop container", e)}}diff --git a/test/src/testFixtures/groovy/org/openremote/test/ManagerContainerTrait.groovy b/test/src/testFixtures/groovy/org/openremote/test/ManagerContainerTrait.groovyindex 23f22479b..2f6d74a9c 100644--- a/test/src/testFixtures/groovy/org/openremote/test/ManagerContainerTrait.groovy+++ b/test/src/testFixtures/groovy/org/openremote/test/ManagerContainerTrait.groovy@@ -18,6 +18,7 @@ import static org.openremote.container.web.WebService.OR_WEBSERVER_LISTEN_PORTimport static org.openremote.manager.mqtt.MQTTBrokerService.MQTT_SERVER_LISTEN_HOSTimport static org.openremote.manager.rules.RulesService.OR_RULES_MIN_TEMP_FACT_EXPIRATION_MILLISimport static org.openremote.manager.rules.RulesService.OR_RULES_QUICK_FIRE_MILLIS+import static org.openremote.model.Container.OR_METRICS_ENABLEDtrait ManagerContainerTrait extends ContainerTrait {@@ -31,7 +32,8 @@ trait ManagerContainerTrait extends ContainerTrait {(MQTTBrokerService.MQTT_FORCE_USER_DISCONNECT_DEBOUNCE_MILLIS): "10",(OR_RULES_QUICK_FIRE_MILLIS): "500",(OR_RULES_MIN_TEMP_FACT_EXPIRATION_MILLIS): "500",- (TIMER_CLOCK_TYPE) : PSEUDO.name()+ (TIMER_CLOCK_TYPE) : PSEUDO.name(),+ (OR_METRICS_ENABLED): "false"]}