100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Add device auto-discovery for ChirpStack and The Things Stack agents (#2317)
rainerhitzb9454cb2f03c
diff --git a/agent/build.gradle b/agent/build.gradleindex 4721af9b8..5d6129569 100644--- a/agent/build.gradle+++ b/agent/build.gradle@@ -48,10 +48,17 @@ dependencies {implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-csv:$jacksonVersion"implementation "com.fasterxml.jackson.core:jackson-databind:$jacksonVersion"+ implementation "io.grpc:grpc-netty-shaded:$grpcVersion"+ implementation "io.grpc:grpc-protobuf:$grpcVersion"+ implementation "io.grpc:grpc-stub:$grpcVersion"+testImplementation "org.junit.jupiter:junit-jupiter-api:$junitVersion"testImplementation "org.mockito:mockito-core:$mockitoVersion"testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junitVersion"testRuntimeOnly "org.junit.platform:junit-platform-launcher"++ implementation files("libs/chirpstack-api-4.14.1.jar")+ implementation files("libs/tts-api-3.34.3.jar")}base {diff --git a/agent/libs/chirpstack-api-4.14.1.jar b/agent/libs/chirpstack-api-4.14.1.jarnew file mode 100644index 000000000..c27151ba2Binary files /dev/null and b/agent/libs/chirpstack-api-4.14.1.jar differdiff --git a/agent/libs/tts-api-3.34.3.jar b/agent/libs/tts-api-3.34.3.jarnew file mode 100644index 000000000..ffa34dcdcBinary files /dev/null and b/agent/libs/tts-api-3.34.3.jar differdiff --git a/agent/src/main/java/org/openremote/agent/protocol/lorawan/AbstractLoRaWANProtocol.java b/agent/src/main/java/org/openremote/agent/protocol/lorawan/AbstractLoRaWANProtocol.javaindex feb6380d9..a66127f0e 100644--- a/agent/src/main/java/org/openremote/agent/protocol/lorawan/AbstractLoRaWANProtocol.java+++ b/agent/src/main/java/org/openremote/agent/protocol/lorawan/AbstractLoRaWANProtocol.java@@ -56,50 +56,52 @@ import java.util.function.Consumer;import java.util.logging.Level;import java.util.logging.Logger;-import static org.openremote.agent.protocol.lorawan.LoRaWANAgent.APPLICATION_ID;-import static org.openremote.agent.protocol.lorawan.LoRaWANAgent.MQTT_HOST;-import static org.openremote.agent.protocol.lorawan.LoRaWANAgent.MQTT_PORT;+import static org.openremote.agent.protocol.lorawan.LoRaWANAgent.*;import static org.openremote.agent.protocol.lorawan.LoRaWANConstants.*;+import static org.openremote.model.asset.agent.Agent.HOST;+import static org.openremote.model.asset.agent.Agent.PORT;import static org.openremote.model.syslog.SyslogCategory.PROTOCOL;+import static org.openremote.model.util.TextUtil.isNullOrEmpty;import static org.openremote.model.value.MetaItemType.AGENT_LINK;import static org.openremote.model.value.MetaItemType.AGENT_LINK_CONFIG;+import static org.openremote.model.value.MetaItemType.STORE_DATA_POINTS;public abstract class AbstractLoRaWANProtocol<S extends AbstractLoRaWANProtocol<S,T>, T extends LoRaWANAgent<T, S>> implements Protocol<T>, ProtocolAssetImport {private static final Logger LOG = SyslogCategory.getLogger(PROTOCOL, AbstractLoRaWANProtocol.class);- private ExecutorService executorService;- private ProtocolAssetService assetService;- private MQTTProtocol mqttProtocol;- private MQTTAgent mqttAgent;- private T agent;- private Set<String> devEuiSet = new CopyOnWriteArraySet<>();- protected Container container;+ protected volatile ExecutorService executorService;+ protected volatile ProtocolAssetService assetService;+ private final MQTTProtocol mqttProtocol;+ private final T agent;+ protected final Set<String> devEuiSet = new CopyOnWriteArraySet<>();+ protected volatile Container container;public AbstractLoRaWANProtocol(T agent) {this.agent = agent;- this.mqttAgent = new MQTTAgent(agent.getName());++ MQTTAgent mqttAgent = new MQTTAgent(agent.getName());mqttAgent.setId(agent.getId());- agent.getMqttHost().ifPresent(host -> mqttAgent.setHost(host));- agent.getMqttPort().ifPresent(port -> mqttAgent.setPort(port));- agent.getClientId().ifPresent(clientId -> mqttAgent.setClientId(clientId));- agent.isSecureMode().ifPresent(secureMode -> mqttAgent.setSecureMode(secureMode));- agent.getPublishQoS().ifPresent(publishQos -> mqttAgent.setPublishQos(publishQos));- agent.getSubscribeQoS().ifPresent(subscribeQos -> mqttAgent.setSubscribeQos(subscribeQos));- agent.getCertificateAlias().ifPresent(certificateAlias -> mqttAgent.setCertificateAlias(certificateAlias));- agent.isResumeSession().ifPresent(resumeSession -> mqttAgent.setResumeSession(resumeSession));- agent.isWebsocketMode().ifPresent(websocketMode -> mqttAgent.setWebsocketMode(websocketMode));- agent.getWebsocketPath().ifPresent(websocketPath -> mqttAgent.setWebsocketPath(websocketPath));- agent.getWebsocketQuery().ifPresent(websocketQuery -> mqttAgent.setWebsocketQuery(websocketQuery));- agent.getLastWillTopic().ifPresent(lastWillTopic -> mqttAgent.setLastWillTopic(lastWillTopic));- agent.getLastWillPayload().ifPresent(lastWillPayload -> mqttAgent.setLastWillPayload(lastWillPayload));- agent.isLastWillRetain().ifPresent(lastWillRetain -> mqttAgent.setLastWillRetain(lastWillRetain));- agent.getUsernamePassword().ifPresent(usernamePassword -> mqttAgent.setUsernamePassword(usernamePassword));+ agent.getMqttHost().ifPresent(mqttAgent::setHost);+ agent.getMqttPort().ifPresent(mqttAgent::setPort);+ agent.getClientId().ifPresent(mqttAgent::setClientId);+ agent.isSecureMode().ifPresent(mqttAgent::setSecureMode);+ agent.getPublishQoS().ifPresent(mqttAgent::setPublishQos);+ agent.getSubscribeQoS().ifPresent(mqttAgent::setSubscribeQos);+ agent.getCertificateAlias().ifPresent(mqttAgent::setCertificateAlias);+ agent.isResumeSession().ifPresent(mqttAgent::setResumeSession);+ agent.isWebsocketMode().ifPresent(mqttAgent::setWebsocketMode);+ agent.getWebsocketPath().ifPresent(mqttAgent::setWebsocketPath);+ agent.getWebsocketQuery().ifPresent(mqttAgent::setWebsocketQuery);+ agent.getLastWillTopic().ifPresent(mqttAgent::setLastWillTopic);+ agent.getLastWillPayload().ifPresent(mqttAgent::setLastWillPayload);+ agent.isLastWillRetain().ifPresent(mqttAgent::setLastWillRetain);+ agent.getUsernamePassword().ifPresent(mqttAgent::setUsernamePassword);mqttAgent.setWildcardSubscriptionTopics(createWildcardSubscriptionTopicList().toArray(new String[0]));- this.mqttProtocol = new LoRaWANMQTTProtocol(mqttAgent);+ this.mqttProtocol = createMqttClientProtocol(mqttAgent);}@Override@@ -116,7 +118,7 @@ public abstract class AbstractLoRaWANProtocol<S extends AbstractLoRaWANProtocol<.map(String::trim).map(String::toUpperCase).filter(s -> !s.isEmpty())- .ifPresent(devEUI -> devEuiSet.add(devEUI));+ .ifPresent(devEuiSet::add);}}@@ -129,7 +131,10 @@ public abstract class AbstractLoRaWANProtocol<S extends AbstractLoRaWANProtocol<.map(String::trim).map(String::toUpperCase).filter(s -> !s.isEmpty())- .ifPresent(devEUI -> devEuiSet.remove(devEUI));+ .ifPresent(devEUI -> {+ devEuiSet.remove(devEUI);+ doUnlinkDevice(devEUI);+ });}}@@ -137,12 +142,6 @@ public abstract class AbstractLoRaWANProtocol<S extends AbstractLoRaWANProtocol<public void start(Container container) throws Exception {this.container = container;executorService = container.getExecutor();- mqttProtocol.start(container);- }-- @Override- public void stop(Container container) throws Exception {- mqttProtocol.stop(container);}@Override@@ -175,6 +174,8 @@ public abstract class AbstractLoRaWANProtocol<S extends AbstractLoRaWANProtocol<public boolean onAgentAttributeChanged(AttributeEvent event) {if (MQTT_HOST.getName().equals(event.getName()) ||MQTT_PORT.getName().equals(event.getName()) ||+ PORT.getName().equals(event.getName()) ||+ HOST.getName().equals(event.getName()) ||APPLICATION_ID.getName().equals(event.getName())){return true;@@ -203,11 +204,11 @@ public abstract class AbstractLoRaWANProtocol<S extends AbstractLoRaWANProtocol<.build();try (ByteArrayInputStream inputStream = new ByteArrayInputStream(fileData)) {- List<CsvRecord> csvEntries = csvMapper.readerFor(CsvRecord.class).with(schema).<CsvRecord>readValues(inputStream).readAll();+ List<DeviceRecord> csvEntries = csvMapper.readerFor(DeviceRecord.class).with(schema).<DeviceRecord>readValues(inputStream).readAll();AssetTreeNode[] assetTreeNodes = csvEntries.stream()- .filter(record -> validateCsvRecord(record))- .filter(record -> duplicateAssetCheck(record))+ .filter(this::validateDeviceRecord)+ .filter(this::duplicateAssetCheck).map(this::createAsset).filter(Optional::isPresent).map(Optional::get)@@ -216,28 +217,50 @@ public abstract class AbstractLoRaWANProtocol<S extends AbstractLoRaWANProtocol<assetConsumer.accept(assetTreeNodes);} catch (IOException e) {- LOG.log(Level.WARNING, "CSV import failed", e);+ LOG.log(Level.WARNING, "CSV import failed for: " + getProtocolInstanceUri(), e);}}, null);}+ protected MQTTProtocol getMqttProtocol() {+ return mqttProtocol;+ }++ protected void startMqttProtocol(Container container) throws Exception {+ if (mqttProtocol != null) {+ mqttProtocol.start(container);+ }+ }++ protected void stopMqttProtocol(Container container) {+ if (mqttProtocol != null) {+ mqttProtocol.stop(container);+ }+ }++ protected void doUnlinkDevice(String devEui) {++ }+protected boolean checkCSVImportPrerequisites() {Optional<String> applicationId = getAgent().getApplicationId();boolean isOk = applicationId.map(id -> !id.trim().isEmpty()).orElse(false);if (!isOk) {- LOG.log(Level.WARNING, "CSV import failed because agent attribute '" + APPLICATION_ID.getName() + "' is missing");+ LOG.warning("CSV import failed because agent attribute '" + APPLICATION_ID.getName() + "' is missing for: " + getProtocolInstanceUri());}return isOk;}+ protected abstract MQTTProtocol createMqttClientProtocol(MQTTAgent agent);protected abstract List<String> createWildcardSubscriptionTopicList();- protected abstract boolean configureMQTTSubscriptionTopic(Attribute<?> attribute, MQTTAgentLink agentLink, CsvRecord csvRecord);- protected abstract boolean configureMQTTPublishTopic(Attribute<?> attribute, MQTTAgentLink agentLink, CsvRecord csvRecord);- protected abstract boolean configureMQTTMessageMatchFilterAndPredicate(Attribute<?> attribute, MQTTAgentLink agentLink, CsvRecord csvRecord);- protected abstract boolean configureMQTTWriteValueTemplate(Attribute<?> attribute, MQTTAgentLink agentLink, CsvRecord csvRecord);-- protected boolean configureMQTTValueFilter(Attribute<?> attribute, MQTTAgentLink agentLink, CsvRecord csvRecord) {- if (attribute == null || agentLink == null || csvRecord == null) {+ protected abstract boolean configureMQTTSubscriptionTopic(Attribute<?> attribute, MQTTAgentLink agentLink, DeviceRecord deviceRecord);+ protected abstract boolean configureMQTTPublishTopic(Attribute<?> attribute, MQTTAgentLink agentLink, DeviceRecord deviceRecord);+ protected abstract boolean configureMQTTMessageMatchFilterAndPredicate(Attribute<?> attribute, MQTTAgentLink agentLink, DeviceRecord deviceRecord);+ protected abstract boolean configureMQTTWriteValueTemplate(Attribute<?> attribute, MQTTAgentLink agentLink, DeviceRecord deviceRecord);+ protected abstract String generateAssetId(String devEui);++ protected boolean configureMQTTValueFilter(Attribute<?> attribute, MQTTAgentLink agentLink, DeviceRecord deviceRecord) {+ if (attribute == null || agentLink == null || deviceRecord == null) {return false;}getAgentConfigJsonPath(attribute).ifPresent(@@ -249,25 +272,25 @@ public abstract class AbstractLoRaWANProtocol<S extends AbstractLoRaWANProtocol<return true;}- protected boolean configureMQTTValueConverter(Attribute<?> attribute, MQTTAgentLink agentLink, CsvRecord csvRecord) {- if (attribute == null || agentLink == null || csvRecord == null) {+ protected boolean configureMQTTValueConverter(Attribute<?> attribute, MQTTAgentLink agentLink, DeviceRecord deviceRecord) {+ if (attribute == null || agentLink == null || deviceRecord == null) {return false;}- getAgentConfigValueConverter(attribute).ifPresent(converter -> agentLink.setValueConverter(converter));+ getAgentConfigValueConverter(attribute).ifPresent(agentLink::setValueConverter);return true;}- protected boolean configureMQTTWriteValueConverter(Attribute<?> attribute, MQTTAgentLink agentLink, CsvRecord csvRecord) {- if (attribute == null || agentLink == null || csvRecord == null) {+ protected boolean configureMQTTWriteValueConverter(Attribute<?> attribute, MQTTAgentLink agentLink, DeviceRecord deviceRecord) {+ if (attribute == null || agentLink == null || deviceRecord == null) {return false;}- getAgentConfigWriteValueConverter(attribute).ifPresent(converter -> agentLink.setWriteValueConverter(converter));+ getAgentConfigWriteValueConverter(attribute).ifPresent(agentLink::setWriteValueConverter);return true;}protected Optional<ValueType.ObjectMap> getAgentConfig(Attribute<?> attribute) {return Optional.ofNullable(attribute)- .map(attr -> attr.getMeta())+ .map(Attribute::getMeta).flatMap(metaMap -> metaMap.get(AGENT_LINK_CONFIG)).flatMap(metItem -> metItem.getValue());}@@ -324,31 +347,31 @@ public abstract class AbstractLoRaWANProtocol<S extends AbstractLoRaWANProtocol<.filter(valueConverter -> !valueConverter.isEmpty());}- private boolean validateCsvRecord(CsvRecord record) {+ private boolean validateDeviceRecord(DeviceRecord record) {boolean isValid = false;if (record != null) {isValid = record.isValid();if (!isValid) {- LOG.log(Level.WARNING, "CSV import skipped an invalid CSV record: " + record);+ LOG.warning("CSV import skipped an invalid device record " + record + " for: " + getProtocolInstanceUri());}}return isValid;}- private boolean duplicateAssetCheck(CsvRecord record) {+ private boolean duplicateAssetCheck(DeviceRecord record) {if (record == null) {return false;}if (!duplicateAssetCheck(record.getDevEUI())) {- LOG.log(Level.INFO, "CSV import skipped a CSV record because an asset already existed: " + record);+ LOG.info("CSV import skipped a device record " + record + " because an asset already existed for: " + getProtocolInstanceUri());return false;}return true;}- private boolean duplicateAssetCheck(String devEUI) {+ protected boolean duplicateAssetCheck(String devEUI) {return Optional.ofNullable(devEUI).map(String::trim).filter(s -> !s.isEmpty())@@ -357,85 +380,95 @@ public abstract class AbstractLoRaWANProtocol<S extends AbstractLoRaWANProtocol<.orElse(false);}- private Optional<Asset<?>> createAsset(CsvRecord csvRecord) {- if (csvRecord == null) {+ protected Optional<Asset<?>> createAsset(DeviceRecord deviceRecord) {+ if (deviceRecord == null) {return Optional.empty();}- return Optional.ofNullable(csvRecord.getAssetTypeName())- .flatMap(name -> resolveAssetClass(name, csvRecord))- .map(clazz -> instantiateAsset(clazz, csvRecord))- .flatMap(asset -> configureAsset(asset, csvRecord));+ return Optional.ofNullable(deviceRecord.getAssetTypeName())+ .flatMap(name -> resolveAssetClass(name, deviceRecord))+ .flatMap(clazz -> instantiateAsset(clazz, deviceRecord))+ .flatMap(asset -> configureAsset(asset, deviceRecord));}- private Optional<Class<? extends Asset<?>>> resolveAssetClass(String simpleClassName, CsvRecord csvRecord) {- if (simpleClassName == null || csvRecord == null) {+ private Optional<Class<? extends Asset<?>>> resolveAssetClass(String simpleClassName, DeviceRecord deviceRecord) {+ if (simpleClassName == null || deviceRecord == null) {return Optional.empty();}return ValueUtil.getAssetClass(simpleClassName).or(() -> {- LOG.log(Level.WARNING, "CSV import skipped a CSV record because of an invalid asset type name: " + csvRecord);+ LOG.warning("CSV import skipped a device record " + deviceRecord + " because of an invalid asset type name for: " + getProtocolInstanceUri());return Optional.empty();});}- private Asset<?> instantiateAsset(Class<?> clazz, CsvRecord csvRecord) {- if (clazz == null || csvRecord == null) {- return null;+ private Optional<Asset<?>> instantiateAsset(Class<?> clazz, DeviceRecord deviceRecord) {+ if (clazz == null || deviceRecord == null) {+ return Optional.empty();}Asset<?> asset = null;try {Constructor<?> constructor = clazz.getConstructor(String.class);- asset = (Asset<?>) constructor.newInstance(csvRecord.getName());+ asset = (Asset<?>) constructor.newInstance(deviceRecord.getName());} catch (ReflectiveOperationException e) {- LOG.log(Level.INFO, "CSV import failed to create asset for CSV record: " + csvRecord, e);+ LOG.log(Level.INFO, "CSV import failed to create asset " + deviceRecord + " for: " + getProtocolInstanceUri(), e);}- return asset;+ return Optional.ofNullable(asset);}- private Optional<Asset<?>> configureAsset(Asset<?> asset, CsvRecord csvRecord) {- if (asset == null || csvRecord == null) {+ protected Optional<Asset<?>> configureAsset(Asset<?> asset, DeviceRecord deviceRecord) {+ if (asset == null || deviceRecord == null) {return Optional.empty();}- Optional.ofNullable(csvRecord.getDevEUI()).ifPresent(devEUI -> asset.getAttribute(ATTRIBUTE_NAME_DEV_EUI).ifPresent(attribute -> attribute.setValue(devEUI.toUpperCase())));- Optional.ofNullable(csvRecord.getVendorId()).ifPresent(vendorId -> asset.getAttribute(ATTRIBUTE_NAME_VENDOR_ID).ifPresent(attribute -> attribute.setValue(vendorId)));- Optional.ofNullable(csvRecord.getModelId()).ifPresent(modelId -> asset.getAttribute(ATTRIBUTE_NAME_MODEL_ID).ifPresent(attribute -> attribute.setValue(modelId)));- Optional.ofNullable(csvRecord.getFirmwareVersion()).ifPresent(version -> asset.getAttribute(ATTRIBUTE_NAME_FIRMWARE_VERSION).ifPresent(attribute -> attribute.setValue(version)));+ Optional.ofNullable(deviceRecord.getDevEUI()).ifPresent(devEUI -> asset.getAttribute(ATTRIBUTE_NAME_DEV_EUI).ifPresent(attribute -> attribute.setValue(devEUI.toUpperCase())));+ Optional.ofNullable(deviceRecord.getVendorId()).ifPresent(vendorId -> asset.getAttribute(ATTRIBUTE_NAME_VENDOR_ID).ifPresent(attribute -> attribute.setValue(vendorId)));+ Optional.ofNullable(deviceRecord.getModelId()).ifPresent(modelId -> asset.getAttribute(ATTRIBUTE_NAME_MODEL_ID).ifPresent(attribute -> attribute.setValue(modelId)));+ Optional.ofNullable(deviceRecord.getFirmwareVersion()).ifPresent(version -> asset.getAttribute(ATTRIBUTE_NAME_FIRMWARE_VERSION).ifPresent(attribute -> attribute.setValue(version)));++ asset.getAttribute(ATTRIBUTE_NAME_DEV_EUI)+ .flatMap(attr -> attr.getValue(String.class))+ .filter(devEui -> !isNullOrEmpty(devEui))+ .ifPresent(devEui -> {+ asset.setId(generateAssetId(devEui));+ asset.setParentId(getAgent().getId());+ });- boolean isOk = true;+ boolean isOk = !isNullOrEmpty(asset.getId());- for (Map.Entry<String, Attribute<?>> entry : asset.getAttributes().entrySet()) {- String name = entry.getKey();- Attribute<?> attribute = entry.getValue();+ for (Attribute<?> attribute : asset.getAttributes().values()) {MQTTAgentLink agentLink = new MQTTAgentLink(agent.getId());Optional<String> jsonPath = getAgentConfigJsonPath(attribute);Optional<String> regex = getAgentConfigRegex(attribute);if (jsonPath.isPresent() || regex.isPresent()) {- isOk = configureMQTTSubscriptionTopic(attribute, agentLink, csvRecord);+ isOk = isOk && configureMQTTSubscriptionTopic(attribute, agentLink, deviceRecord);}- isOk = isOk && configureMQTTValueFilter(attribute, agentLink, csvRecord);- isOk = isOk && configureMQTTMessageMatchFilterAndPredicate(attribute, agentLink, csvRecord);- isOk = isOk && configureMQTTValueConverter(attribute, agentLink, csvRecord);+ isOk = isOk && configureMQTTValueFilter(attribute, agentLink, deviceRecord);+ isOk = isOk && configureMQTTMessageMatchFilterAndPredicate(attribute, agentLink, deviceRecord);+ isOk = isOk && configureMQTTValueConverter(attribute, agentLink, deviceRecord);Optional<Integer> downlinkPort = getAgentConfigDownlinkPort(attribute);if (downlinkPort.isPresent()) {- isOk = isOk && configureMQTTPublishTopic(attribute, agentLink, csvRecord);- isOk = isOk && configureMQTTWriteValueConverter(attribute, agentLink, csvRecord);- isOk = isOk && configureMQTTWriteValueTemplate(attribute, agentLink, csvRecord);+ isOk = isOk && configureMQTTPublishTopic(attribute, agentLink, deviceRecord);+ isOk = isOk && configureMQTTWriteValueConverter(attribute, agentLink, deviceRecord);+ isOk = isOk && configureMQTTWriteValueTemplate(attribute, agentLink, deviceRecord);}- if (getAgentConfig(attribute).isPresent() || ATTRIBUTE_NAME_DEV_EUI.equals(attribute.getName())) {+ if (getAgentConfig(attribute).isPresent()) {attribute.addOrReplaceMeta(new MetaItem<>(AGENT_LINK, agentLink));+ MetaMap metaMap = attribute.getMeta();+ metaMap.remove(AGENT_LINK_CONFIG);+ attribute.setMeta(metaMap);+ } else if (ATTRIBUTE_NAME_DEV_EUI.equals(attribute.getName())) {+ attribute.addOrReplaceMeta(+ new MetaItem<>(AGENT_LINK, agentLink),+ new MetaItem<>(STORE_DATA_POINTS, false)+ );}-- MetaMap metaMap = attribute.getMeta();- metaMap.remove(AGENT_LINK_CONFIG);- attribute.setMeta(metaMap);}return isOk ? Optional.of(asset) : Optional.empty();diff --git a/agent/src/main/java/org/openremote/agent/protocol/lorawan/CsvRecord.java b/agent/src/main/java/org/openremote/agent/protocol/lorawan/DeviceRecord.javasimilarity index 97%rename from agent/src/main/java/org/openremote/agent/protocol/lorawan/CsvRecord.javarename to agent/src/main/java/org/openremote/agent/protocol/lorawan/DeviceRecord.javaindex fee682c61..90e002aee 100644--- a/agent/src/main/java/org/openremote/agent/protocol/lorawan/CsvRecord.java+++ b/agent/src/main/java/org/openremote/agent/protocol/lorawan/DeviceRecord.java@@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;import java.util.Optional;-public class CsvRecord {+public class DeviceRecord {@JsonProperty("devEUI")private String devEUI;@@ -68,7 +68,7 @@ public class CsvRecord {@Overridepublic String toString() {- return "CsvRecord{" ++ return "DeviceRecord{" +"devEUI='" + (devEUI != null ? devEUI : "") + '\'' +", name='" + (name != null ? name : "") + '\'' +", vendorId='" + (vendorId != null ? vendorId : "") + '\'' +diff --git a/agent/src/main/java/org/openremote/agent/protocol/lorawan/GrpcRetryingRunner.java b/agent/src/main/java/org/openremote/agent/protocol/lorawan/GrpcRetryingRunner.javanew file mode 100644index 000000000..f9cd29d03--- /dev/null+++ b/agent/src/main/java/org/openremote/agent/protocol/lorawan/GrpcRetryingRunner.java@@ -0,0 +1,317 @@+/*+ * Copyright 2025, OpenRemote Inc.+ *+ * See the CONTRIBUTORS.txt file in the distribution for a+ * full listing of individual contributors.+ *+ * This program is free software: you can redistribute it and/or modify+ * it under the terms of the GNU Affero General Public License as+ * published by the Free Software Foundation, either version 3 of the+ * License, or (at your option) any later version.+ *+ * This program is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+ * GNU Affero General Public License for more details.+ *+ * You should have received a copy of the GNU Affero General Public License+ * along with this program. If not, see <http://www.gnu.org/licenses/>.+ */+package org.openremote.agent.protocol.lorawan;++import dev.failsafe.Failsafe;+import dev.failsafe.RetryPolicy;+import dev.failsafe.function.CheckedBiPredicate;+import io.grpc.ManagedChannel;+import io.grpc.Status;+import io.grpc.StatusRuntimeException;+import org.openremote.container.Container;+import org.openremote.model.asset.agent.ConnectionStatus;+import org.openremote.model.syslog.SyslogCategory;++import java.io.IOException;+import java.time.Duration;+import java.util.concurrent.CancellationException;+import java.util.concurrent.CompletableFuture;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.function.Supplier;+import java.util.logging.Level;+import java.util.logging.Logger;++import static org.openremote.model.syslog.SyslogCategory.PROTOCOL;++public class GrpcRetryingRunner {++ public static final Logger LOG = SyslogCategory.getLogger(PROTOCOL, GrpcRetryingRunner.class);++ private final String taskName;+ private final Supplier<String> clientUriSupplier;+ private volatile CompletableFuture<Void> future;+ private volatile ManagedChannel currentChannel;+ private volatile Future<?> scheduledContinuationFuture;++ public static boolean isNonRetryableCode(StatusRuntimeException exception) {+ if (exception != null) {+ Status.Code code = exception.getStatus().getCode();++ return code == Status.Code.NOT_FOUND ||+ code == Status.Code.INVALID_ARGUMENT ||+ code == Status.Code.UNAUTHENTICATED ||+ code == Status.Code.FAILED_PRECONDITION ||+ code == Status.Code.CANCELLED;+ }+ return false;+ }++ public GrpcRetryingRunner(String taskName, Supplier<String> clientUriSupplier) {+ this.taskName = taskName;+ this.clientUriSupplier = clientUriSupplier != null ? clientUriSupplier : () -> "";+ }++ public void start(Supplier<ManagedChannel> channelSupplier,+ ThrowingConsumer<ManagedChannel> task,+ boolean isVerboseLog,+ Duration initialBackoff,+ Duration maxBackoff)+ {+ start(channelSupplier, task, isVerboseLog, initialBackoff, maxBackoff, null);+ }++ public void start(Supplier<ManagedChannel> channelSupplier,+ ThrowingConsumer<ManagedChannel> task,+ boolean isVerboseLog,+ Duration initialBackoff,+ Duration maxBackoff,+ Duration scheduleInterval)+ {+ RunnableContinuation continuation;++ if (scheduleInterval != null) {+ continuation = () -> {+ scheduleNextStart(channelSupplier, task, isVerboseLog, initialBackoff, maxBackoff, scheduleInterval);+ };+ } else {+ continuation = () -> {};+ }++ CheckedBiPredicate<Object, Throwable> shouldNotRetry = (result, ex) -> {+ if (ex instanceof StatusRuntimeException sre) {+ return isNonRetryableCode(sre);+ }+ // Note: CancellationException is an IllegalStateException+ return ex instanceof IllegalStateException;+ };++ LOG.log(isVerboseLog ? Level.INFO: Level.FINE, "[" + taskName + "] Establishing gRPC connection: " + getClientUri());++ RetryPolicy<Object> retryPolicy = RetryPolicy.builder()+ .handle(Exception.class)+ .abortIf(shouldNotRetry)+ .withBackoff(initialBackoff, maxBackoff)+ .withJitter(initialBackoff.minusMillis(1))+ .onRetryScheduled(e -> LOG.log(isVerboseLog ? Level.INFO: Level.FINE,() -> "[" + taskName + "] " + "Re-connection scheduled in '" + e.getDelay() + "' for: " + getClientUri()))+ .onFailedAttempt(e -> LOG.log(isVerboseLog ? Level.INFO: Level.FINE,() -> "[" + taskName + "] " + "Connection attempt failed '" + e.getAttemptCount() + "' for: " + getClientUri() + ", error=" + (e.getLastException() != null ? e.getLastException().getMessage() : null)))+ .onSuccess(e -> LOG.fine("[" + taskName + "] gRPC connection attempt success: " + getClientUri()))+ .withMaxRetries(Integer.MAX_VALUE)+ .build();++ future = Failsafe+ .with(retryPolicy)+ .with(Container.EXECUTOR)+ .runAsyncExecution(execution -> {+ LOG.fine(() ->+ "[" + taskName + "] Connection attempt '" + (execution.getAttemptCount()+1) + "' for: " + getClientUri()+ );+ ManagedChannel channel = channelSupplier.get();+ currentChannel = channel;+ try {+ task.accept(channel);+ execution.recordResult(null);+ } catch (InterruptedException ex) {+ throw new CancellationException("[" + taskName + "] cancelled");+ } finally {+ shutdownChannel(channel);+ currentChannel = null;+ }+ });+ future.whenComplete((result, ex) -> {+ if (ex instanceof CancellationException) {+ LOG.info("[" + taskName + "] gRPC connection shutdown for: " + getClientUri());+ } else if (ex != null) {+ LOG.warning("[" + taskName + "] gRPC connection unexpectedly stopped for: " + getClientUri() + ", error=" + ex.getMessage());+ } else {+ try {+ continuation.run();+ } catch (Exception e) {+ LOG.warning("[" + taskName + "] Error during continuation for: " + getClientUri() + ", error=" + e.getMessage());+ }+ }+ });+ }++ public void startStream(Supplier<ManagedChannel> channelSupplier,+ ThrowingConsumer<ManagedChannel> task,+ Duration initialBackoff,+ Duration maxBackoff,+ ConnectionStatusObserver observer) {++ Runnable notifyConnecting = () -> { if(observer != null) observer.onStatusChange(ConnectionStatus.CONNECTING); };+ Runnable notifyError = () -> { if(observer != null) observer.onStatusChange(ConnectionStatus.ERROR); };++ CheckedBiPredicate<Object, Throwable> permanentAbort = (result, ex) -> {+ if (ex instanceof StatusRuntimeException sre) {+ return isNonRetryableCode(sre);+ }+ // Note: CancellationException is an IllegalStateException+ return ex instanceof IllegalStateException;+ };++ CheckedBiPredicate<Object, Throwable> innerAbort = (result, ex) ->+ ex instanceof StreamBreakSignalException || permanentAbort.test(result, ex);++ RetryPolicy<Object> innerPolicy = RetryPolicy.builder()+ .handle(Exception.class)+ .abortIf(innerAbort)+ .withBackoff(initialBackoff, maxBackoff)+ .withJitter(initialBackoff.minusMillis(1))+ .onRetryScheduled(e -> LOG.info("[" + taskName + "] " + "Re-connection scheduled in '" + e.getDelay() + "' for: " + getClientUri()))+ .onFailedAttempt(e -> {+ Throwable ex = e.getLastException();+ if (ex instanceof StreamBreakSignalException streamException) {+ Throwable cause = streamException.getCause();+ LOG.info("[" + taskName + "] " + "gRPC stream failure for: " + getClientUri() + ", error=" + (cause != null ? cause.getMessage() : streamException.getMessage()));+ } else if (!isCancelledException(e.getLastException())) {+ LOG.info("[" + taskName + "] " + "Connection attempt failed '" + e.getAttemptCount() + "' for: " + getClientUri() + ", error=" + (e.getLastException() != null ? e.getLastException().getMessage() : null));+ }+ })+ .withMaxRetries(Integer.MAX_VALUE)+ .build();+++ RetryPolicy<Object> outerPolicy = RetryPolicy.builder()+ .handle(Exception.class)+ .abortIf(permanentAbort)+ .withMaxRetries(Integer.MAX_VALUE)+ .withDelay(Duration.ofSeconds(1))+ .build();++ future = Failsafe+ .with(outerPolicy)+ .with(Container.EXECUTOR)+ .runAsync(() -> {+ Failsafe+ .with(innerPolicy)+ .with(Container.EXECUTOR)+ .run(() -> {+ ManagedChannel channel = channelSupplier.get();+ currentChannel = channel;+ try {+ notifyConnecting.run();+ LOG.info("[" + taskName + "] Establishing gRPC stream connection for: " + getClientUri());+ task.accept(channel);+ } catch (InterruptedException exception) {+ throw new CancellationException("[" + taskName + "] cancelled for: " + getClientUri());+ } finally {+ shutdownChannel(channel);+ currentChannel = null;+ }+ });+ });++ future.whenComplete((result, ex) -> {+ if (ex instanceof CancellationException) {+ LOG.info("[" + taskName + "] gRPC stream connection shutdown for: " + getClientUri());+ } else if (ex != null) {+ LOG.warning("[" + taskName + "] gRPC stream connection unexpectedly stopped for: " + getClientUri() + ", error=" + ex.getMessage());+ notifyError.run();+ }+ });+ }++ public void shutdown() {+ if (future != null && !future.isDone()) {+ future.cancel(true);+ }++ if (scheduledContinuationFuture != null) {+ scheduledContinuationFuture.cancel(false);+ scheduledContinuationFuture = null;+ }++ if (currentChannel != null) {+ shutdownChannel(currentChannel);+ }+ }++ private void shutdownChannel(ManagedChannel channel) {+ if (channel == null) {+ return;+ }+ try {+ channel.shutdown();+ if (!channel.awaitTermination(3, TimeUnit.SECONDS)) {+ channel.shutdownNow();+ }+ } catch (InterruptedException e) {+ channel.shutdownNow();+ Thread.currentThread().interrupt();+ }+ }++ private String getClientUri() {+ return clientUriSupplier.get();+ }++ private void scheduleNextStart(Supplier<ManagedChannel> channelSupplier,+ ThrowingConsumer<ManagedChannel> task,+ boolean isVerboseLog,+ Duration initialBackoff,+ Duration maxBackoff,+ Duration scheduleInterval) {++ if (scheduledContinuationFuture != null) {+ scheduledContinuationFuture.cancel(false);+ }++ Runnable delayedTask = () -> {+ this.start(channelSupplier, task, isVerboseLog, initialBackoff, maxBackoff, scheduleInterval);+ this.scheduledContinuationFuture = null;+ };++ this.scheduledContinuationFuture = Container.SCHEDULED_EXECUTOR.schedule(+ delayedTask,+ scheduleInterval.toMillis(),+ TimeUnit.MILLISECONDS+ );+ LOG.info(() ->+ "[" + taskName + "] Scheduled next execution in '" + scheduleInterval + "' for: " + getClientUri()+ );+ }++ private boolean isCancelledException(Throwable exception) {+ return (exception instanceof CancellationException) ||+ ((exception instanceof StatusRuntimeException) && (((StatusRuntimeException)exception).getStatus().getCode() == Status.Code.CANCELLED));+ }++ @FunctionalInterface+ public interface ThrowingConsumer<T> {+ void accept(T value) throws Exception;+ }++ @FunctionalInterface+ public interface RunnableContinuation {+ void run();+ }++ @FunctionalInterface+ public interface ConnectionStatusObserver {+ void onStatusChange(ConnectionStatus status);+ }++ public static class StreamBreakSignalException extends IOException {+ public StreamBreakSignalException(String message, Throwable cause) {+ super(message, cause);+ }+ }+}diff --git a/agent/src/main/java/org/openremote/agent/protocol/lorawan/LoRaWANAgent.java b/agent/src/main/java/org/openremote/agent/protocol/lorawan/LoRaWANAgent.javaindex bef9bf341..979474817 100644--- a/agent/src/main/java/org/openremote/agent/protocol/lorawan/LoRaWANAgent.java+++ b/agent/src/main/java/org/openremote/agent/protocol/lorawan/LoRaWANAgent.java@@ -47,6 +47,7 @@ public abstract class LoRaWANAgent<T extends LoRaWANAgent<T, U>, U extends Abstrpublic static final AttributeDescriptor<Boolean> LAST_WILL_RETAIN = MQTTAgent.LAST_WILL_RETAIN;public static final AttributeDescriptor<String> APPLICATION_ID = new AttributeDescriptor<>("applicationId", ValueType.TEXT);+ public static final AttributeDescriptor<String> API_KEY = new AttributeDescriptor<>("apiKey", ValueType.TEXT);// For Hydrators@@ -190,4 +191,13 @@ public abstract class LoRaWANAgent<T extends LoRaWANAgent<T, U>, U extends AbstrgetAttributes().getOrCreate(APPLICATION_ID).setValue(applicationId);return this;}++ public Optional<String> getApiKey() {+ return getAttributes().getValue(API_KEY);+ }++ public LoRaWANAgent<T, U> setApiKey(String apiKey) {+ getAttributes().getOrCreate(API_KEY).setValue(apiKey);+ return this;+ }}diff --git a/agent/src/main/java/org/openremote/agent/protocol/lorawan/LoRaWANConstants.java b/agent/src/main/java/org/openremote/agent/protocol/lorawan/LoRaWANConstants.javaindex b4bdc12e1..cd7b5b8c2 100644--- a/agent/src/main/java/org/openremote/agent/protocol/lorawan/LoRaWANConstants.java+++ b/agent/src/main/java/org/openremote/agent/protocol/lorawan/LoRaWANConstants.java@@ -32,4 +32,6 @@ public final class LoRaWANConstants {public static final String ATTRIBUTE_NAME_VENDOR_ID = "vendorId";public static final String ATTRIBUTE_NAME_MODEL_ID = "modelId";public static final String ATTRIBUTE_NAME_FIRMWARE_VERSION= "firmwareVersion";++ public static final String ASSET_TYPE_TAG = "openremote-asset-type";}diff --git a/agent/src/main/java/org/openremote/agent/protocol/lorawan/LoRaWANMQTTProtocol.java b/agent/src/main/java/org/openremote/agent/protocol/lorawan/LoRaWANMQTTProtocol.javaindex 6582a19ae..b59b44cd8 100644--- a/agent/src/main/java/org/openremote/agent/protocol/lorawan/LoRaWANMQTTProtocol.java+++ b/agent/src/main/java/org/openremote/agent/protocol/lorawan/LoRaWANMQTTProtocol.java@@ -21,9 +21,14 @@ package org.openremote.agent.protocol.lorawan;import org.openremote.agent.protocol.mqtt.MQTTAgent;import org.openremote.agent.protocol.mqtt.MQTTProtocol;+import org.openremote.model.asset.agent.ConnectionStatus;public class LoRaWANMQTTProtocol extends MQTTProtocol {protected LoRaWANMQTTProtocol(MQTTAgent agent) {super(agent);}++ public void setStatus(ConnectionStatus status) {+ setConnectionStatus(status);+ }}diff --git a/agent/src/main/java/org/openremote/agent/protocol/lorawan/UniqueBlockingQueue.java b/agent/src/main/java/org/openremote/agent/protocol/lorawan/UniqueBlockingQueue.javanew file mode 100644index 000000000..a2c284e9f--- /dev/null+++ b/agent/src/main/java/org/openremote/agent/protocol/lorawan/UniqueBlockingQueue.java@@ -0,0 +1,86 @@+/*+ * Copyright 2025, OpenRemote Inc.+ *+ * See the CONTRIBUTORS.txt file in the distribution for a+ * full listing of individual contributors.+ *+ * This program is free software: you can redistribute it and/or modify+ * it under the terms of the GNU Affero General Public License as+ * published by the Free Software Foundation, either version 3 of the+ * License, or (at your option) any later version.+ *+ * This program is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+ * GNU Affero General Public License for more details.+ *+ * You should have received a copy of the GNU Affero General Public License+ * along with this program. If not, see <http://www.gnu.org/licenses/>.+ */+package org.openremote.agent.protocol.lorawan;++import java.util.HashSet;+import java.util.Set;+import java.util.concurrent.BlockingQueue;+import java.util.concurrent.LinkedBlockingQueue;+import java.util.concurrent.locks.ReentrantLock;++public class UniqueBlockingQueue<T> {++ private final BlockingQueue<T> queue = new LinkedBlockingQueue<>();+ private final Set<T> set = new HashSet<>();+ private final ReentrantLock lock = new ReentrantLock();++ public boolean put(T item) throws InterruptedException {+ lock.lock();+ try {+ if (set.add(item)) {+ queue.put(item);+ return true;+ }+ return false;+ } finally {+ lock.unlock();+ }+ }++ public T take() throws InterruptedException {+ T item = queue.take();++ lock.lock();+ try {+ set.remove(item);+ return item;+ } finally {+ lock.unlock();+ }+ }++ public T peek() {+ lock.lock();+ try {+ return queue.peek();+ } finally {+ lock.unlock();+ }+ }++ public int size() {+ lock.lock();+ try {+ return queue.size();+ } finally {+ lock.unlock();+ }+ }++ public void clear() {+ lock.lock();+ try {+ queue.clear();+ set.clear();+ } finally {+ lock.unlock();+ }+ }+}diff --git a/agent/src/main/java/org/openremote/agent/protocol/lorawan/chirpstack/ChirpStackAgent.java b/agent/src/main/java/org/openremote/agent/protocol/lorawan/chirpstack/ChirpStackAgent.javaindex 2c7175030..6a375b530 100644--- a/agent/src/main/java/org/openremote/agent/protocol/lorawan/chirpstack/ChirpStackAgent.java+++ b/agent/src/main/java/org/openremote/agent/protocol/lorawan/chirpstack/ChirpStackAgent.java@@ -22,11 +22,20 @@ package org.openremote.agent.protocol.lorawan.chirpstack;import jakarta.persistence.Entity;import org.openremote.agent.protocol.lorawan.LoRaWANAgent;import org.openremote.agent.protocol.mqtt.MQTTAgentLink;+import org.openremote.model.asset.agent.Agent;import org.openremote.model.asset.agent.AgentDescriptor;+import org.openremote.model.value.AttributeDescriptor;+import org.openremote.model.value.ValueType;++import java.util.Optional;@Entitypublic class ChirpStackAgent extends LoRaWANAgent<ChirpStackAgent, ChirpStackProtocol> {+ public static final AttributeDescriptor<String> HOST = Agent.HOST.withOptional(false);+ public static final AttributeDescriptor<Integer> PORT = Agent.PORT.withOptional(false);+ public static final AttributeDescriptor<Boolean> SECURE_GRPC = new AttributeDescriptor<>("secureGRPC", ValueType.BOOLEAN).withOptional(false);+public static final AgentDescriptor<ChirpStackAgent, ChirpStackProtocol, MQTTAgentLink> DESCRIPTOR = new AgentDescriptor<>(ChirpStackAgent.class, ChirpStackProtocol.class, MQTTAgentLink.class);@@ -45,4 +54,13 @@ public class ChirpStackAgent extends LoRaWANAgent<ChirpStackAgent, ChirpStackPropublic ChirpStackProtocol getProtocolInstance() {return new ChirpStackProtocol(this);}++ public Optional<Boolean> getSecureGRPC() {+ return getAttributes().getValue(SECURE_GRPC);+ }++ public ChirpStackAgent setSecureGRPC(Boolean secureGRPC) {+ getAttributes().getOrCreate(SECURE_GRPC).setValue(secureGRPC);+ return this;+ }}diff --git a/agent/src/main/java/org/openremote/agent/protocol/lorawan/chirpstack/ChirpStackProtocol.java b/agent/src/main/java/org/openremote/agent/protocol/lorawan/chirpstack/ChirpStackProtocol.javaindex c727420ab..d93f54a98 100644--- a/agent/src/main/java/org/openremote/agent/protocol/lorawan/chirpstack/ChirpStackProtocol.java+++ b/agent/src/main/java/org/openremote/agent/protocol/lorawan/chirpstack/ChirpStackProtocol.java@@ -19,22 +19,70 @@*/package org.openremote.agent.protocol.lorawan.chirpstack;+import io.chirpstack.api.DeviceListItem;+import io.chirpstack.api.DeviceProfile;+import io.chirpstack.api.DeviceProfileServiceGrpc;+import io.chirpstack.api.DeviceServiceGrpc;+import io.chirpstack.api.GetDeviceProfileRequest;+import io.chirpstack.api.GetDeviceProfileResponse;+import io.chirpstack.api.ListDevicesRequest;+import io.chirpstack.api.ListDevicesResponse;+import io.grpc.ManagedChannel;+import io.grpc.ManagedChannelBuilder;+import io.grpc.Metadata;+import io.grpc.Status;+import io.grpc.StatusRuntimeException;+import io.grpc.stub.MetadataUtils;import org.openremote.agent.protocol.lorawan.AbstractLoRaWANProtocol;-import org.openremote.agent.protocol.lorawan.CsvRecord;+import org.openremote.agent.protocol.lorawan.DeviceRecord;+import org.openremote.agent.protocol.lorawan.LoRaWANMQTTProtocol;+import org.openremote.agent.protocol.mqtt.MQTTAgent;import org.openremote.agent.protocol.mqtt.MQTTAgentLink;+import org.openremote.agent.protocol.mqtt.MQTTProtocol;+import org.openremote.model.Container;+import org.openremote.model.asset.Asset;+import org.openremote.model.asset.AssetTreeNode;import org.openremote.model.attribute.Attribute;+import org.openremote.model.attribute.AttributeEvent;+import org.openremote.model.protocol.ProtocolAssetDiscovery;import org.openremote.model.query.filter.NumberPredicate;+import org.openremote.model.syslog.SyslogCategory;+import org.openremote.model.util.UniqueIdentifierGenerator;+import org.openremote.model.util.ValueUtil;+import org.openremote.model.value.AttributeDescriptor;import org.openremote.model.value.JsonPathFilter;import org.openremote.model.value.ValueFilter;+import java.lang.reflect.Constructor;+import java.util.AbstractMap;import java.util.ArrayList;import java.util.Collections;import java.util.List;+import java.util.Map;import java.util.Optional;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.function.Consumer;+import java.util.logging.Level;+import java.util.logging.Logger;+import java.util.stream.Collectors;-public class ChirpStackProtocol extends AbstractLoRaWANProtocol<ChirpStackProtocol, ChirpStackAgent> {+import static org.openremote.agent.protocol.lorawan.LoRaWANAgent.API_KEY;+import static org.openremote.agent.protocol.lorawan.LoRaWANAgent.APPLICATION_ID;+import static org.openremote.agent.protocol.lorawan.LoRaWANConstants.ASSET_TYPE_TAG;+import static org.openremote.agent.protocol.lorawan.chirpstack.ChirpStackAgent.SECURE_GRPC;+import static org.openremote.model.asset.agent.Agent.HOST;+import static org.openremote.model.syslog.SyslogCategory.PROTOCOL;+import static org.openremote.model.util.TextUtil.isNullOrEmpty;++public class ChirpStackProtocol extends AbstractLoRaWANProtocol<ChirpStackProtocol, ChirpStackAgent> implements ProtocolAssetDiscovery {++ private static final Logger LOG = SyslogCategory.getLogger(PROTOCOL, ChirpStackProtocol.class);public static final String PROTOCOL_DISPLAY_NAME = "ChirpStack";+ public static final String CHIRPSTACK_ASSET_TYPE_TAG = ASSET_TYPE_TAG;+ public static final int GRPC_LIST_DEVICE_LIMIT = 10000;+ public static final long GRPC_TIMEOUT_MILLIS = 10000L;public ChirpStackProtocol(ChirpStackAgent agent) {super(agent);@@ -47,8 +95,76 @@ public class ChirpStackProtocol extends AbstractLoRaWANProtocol<ChirpStackProtoc@Overridepublic String getProtocolInstanceUri() {- return "chirpstack://" + getAgent().getMqttHost().orElse("-") + ":"- + getAgent().getMqttPort().map(p -> p.toString()).orElse("-");+ return "chirpstack-mqtt://" + getAgent().getMqttHost().orElse("-") + ":" + getAgent().getMqttPort().map(Object::toString).orElse("-")+ + "/?clientId=" + getAgent().getClientId().orElse("-");+ }++ @Override+ public void start(Container container) throws Exception {+ super.start(container);+ startMqttProtocol(container);+ }++ @Override+ public void stop(Container container) throws Exception {+ stopMqttProtocol(container);+ }++ @Override+ public boolean onAgentAttributeChanged(AttributeEvent event) {+ if (API_KEY.getName().equals(event.getName()) ||+ SECURE_GRPC.getName().equals(event.getName()))+ {+ return true;+ }++ return super.onAgentAttributeChanged(event);+ }++ @Override+ protected MQTTProtocol createMqttClientProtocol(MQTTAgent agent) {+ return new LoRaWANMQTTProtocol(agent) {+ @Override+ public String getProtocolName() {+ return "ChirpStack MQTT Client";+ }+ };+ }++ @Override+ public Future<Void> startAssetDiscovery(Consumer<AssetTreeNode[]> assetConsumer) {+ return executorService.submit(() -> {+ if (!checkAutoDiscoveryPrerequisites()) {+ assetConsumer.accept(new AssetTreeNode[0]);+ return;+ }++ try {+ AssetTreeNode[] assetTreeNodes = discoverDevices();+ assetConsumer.accept(assetTreeNodes);+ LOG.info("Auto-discovery added '" + assetTreeNodes.length + "' assets for: " + getGRPCClientUri());+ } catch (Exception e) {+ LOG.log(Level.WARNING, "Auto-discovery failed for: " + getGRPCClientUri(), e);+ }+ }, null);+ }++ protected boolean checkAutoDiscoveryPrerequisites() {+ List<AbstractMap.SimpleEntry<AttributeDescriptor<?>, Optional<String>>> list = new ArrayList<>(2);+ list.add(new AbstractMap.SimpleEntry<>(HOST, getAgent().getHost()));+ list.add(new AbstractMap.SimpleEntry<>(APPLICATION_ID, getAgent().getApplicationId()));+ list.add(new AbstractMap.SimpleEntry<>(API_KEY, getAgent().getApiKey()));++ boolean isOk = true;++ for (AbstractMap.SimpleEntry<AttributeDescriptor<?>, Optional<String>> item : list) {+ if (!item.getValue().map(attrValue -> !attrValue.trim().isEmpty()).orElse(false)) {+ isOk = false;+ LOG.warning("Auto-discovery failed because agent attribute '" + item.getKey().getName() + "' is missing for: " + getGRPCClientUri());+ }+ }++ return isOk;}@Override@@ -59,13 +175,13 @@ public class ChirpStackProtocol extends AbstractLoRaWANProtocol<ChirpStackProtoc}@Override- protected boolean configureMQTTSubscriptionTopic(Attribute<?> attribute, MQTTAgentLink agentLink, CsvRecord csvRecord) {- if (attribute == null || agentLink == null || csvRecord == null) {+ protected boolean configureMQTTSubscriptionTopic(Attribute<?> attribute, MQTTAgentLink agentLink, DeviceRecord deviceRecord) {+ if (attribute == null || agentLink == null || deviceRecord == null) {return false;}- Optional<String> applicationId = getAgent().getApplicationId().map(id -> id.trim());- Optional<String> devEUI = Optional.ofNullable(csvRecord.getDevEUI()).map(eui -> eui.trim());+ Optional<String> applicationId = getAgent().getApplicationId().map(String::trim);+ Optional<String> devEUI = Optional.ofNullable(deviceRecord.getDevEUI()).map(String::trim);if (applicationId.isPresent() && devEUI.isPresent()) {agentLink.setSubscriptionTopic("application/" + applicationId.get() + "/device/" + devEUI.get().toLowerCase() + "/event/up");@@ -74,13 +190,13 @@ public class ChirpStackProtocol extends AbstractLoRaWANProtocol<ChirpStackProtoc}@Override- protected boolean configureMQTTPublishTopic(Attribute<?> attribute, MQTTAgentLink agentLink, CsvRecord csvRecord) {- if (attribute == null || agentLink == null || csvRecord == null) {+ protected boolean configureMQTTPublishTopic(Attribute<?> attribute, MQTTAgentLink agentLink, DeviceRecord deviceRecord) {+ if (attribute == null || agentLink == null || deviceRecord == null) {return false;}- Optional<String> applicationId = getAgent().getApplicationId().map(id -> id.trim());- Optional<String> devEUI = Optional.ofNullable(csvRecord.getDevEUI()).map(eui -> eui.trim());+ Optional<String> applicationId = getAgent().getApplicationId().map(String::trim);+ Optional<String> devEUI = Optional.ofNullable(deviceRecord.getDevEUI()).map(String::trim);if (applicationId.isPresent() && devEUI.isPresent()) {agentLink.setPublishTopic("application/" + applicationId.get() + "/device/" + devEUI.get().toLowerCase() + "/command/down");@@ -89,8 +205,8 @@ public class ChirpStackProtocol extends AbstractLoRaWANProtocol<ChirpStackProtoc}@Override- protected boolean configureMQTTMessageMatchFilterAndPredicate(Attribute<?> attribute, MQTTAgentLink agentLink, CsvRecord csvRecord) {- if (attribute == null || agentLink == null || csvRecord == null) {+ protected boolean configureMQTTMessageMatchFilterAndPredicate(Attribute<?> attribute, MQTTAgentLink agentLink, DeviceRecord deviceRecord) {+ if (attribute == null || agentLink == null || deviceRecord == null) {return false;}getAgentConfigUplinkPort(attribute).ifPresent(port ->@@ -101,13 +217,13 @@ public class ChirpStackProtocol extends AbstractLoRaWANProtocol<ChirpStackProtoc}@Override- protected boolean configureMQTTWriteValueTemplate(Attribute<?> attribute, MQTTAgentLink agentLink, CsvRecord csvRecord) {- if (attribute == null || agentLink == null || csvRecord == null) {+ protected boolean configureMQTTWriteValueTemplate(Attribute<?> attribute, MQTTAgentLink agentLink, DeviceRecord deviceRecord) {+ if (attribute == null || agentLink == null || deviceRecord == null) {return false;}Optional<Integer> downlinkPort = getAgentConfigDownlinkPort(attribute);- Optional<String> devEUI = Optional.ofNullable(csvRecord.getDevEUI()).map(eui -> eui.trim());+ Optional<String> devEUI = Optional.ofNullable(deviceRecord.getDevEUI()).map(String::trim);Optional<String> objectTemplate = getAgentConfigWriteObjectValueTemplate(attribute);if (downlinkPort.isPresent() && devEUI.isPresent()) {@@ -125,4 +241,246 @@ public class ChirpStackProtocol extends AbstractLoRaWANProtocol<ChirpStackProtoc}return true;}++ @Override+ protected String generateAssetId(String devEui) {+ return UniqueIdentifierGenerator.generateId("ChirpStack_" + getAgent().getId() + devEui);+ }++ private AssetTreeNode[] discoverDevices() {+ Optional<String> host = getAgent().getHost().map(String::trim);+ int port = getPort();+ boolean isSecureGRPC = isSecureGRPC();+ Optional<String> apiKey = getAgent().getApiKey().map(String::trim);+ Optional<String> applicationId = getAgent().getApplicationId().map(String::trim);++ if (host.isEmpty() || apiKey.isEmpty() || applicationId.isEmpty()) {+ return new AssetTreeNode[0];+ }++ ManagedChannel channel = createChannel(host.get(), port, isSecureGRPC);++ Metadata headers = new Metadata();+ Metadata.Key<String> AUTHORIZATION_HEADER = Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER);+ headers.put(AUTHORIZATION_HEADER, "Bearer " + apiKey.get());++ try {+ List<DeviceListItem> deviceList = requestDeviceList(applicationId.get(), channel, headers);++ Map<String, DeviceProfile> profileMap = deviceList.stream()+ .map(DeviceListItem::getDeviceProfileId)+ .filter(id -> !isNullOrEmpty(id))+ .distinct()+ .map(id -> Map.entry(id, requestDeviceProfile(id, channel, headers)))+ .filter(entry -> entry.getValue().isPresent())+ .collect(Collectors.toMap(+ Map.Entry::getKey,+ entry -> entry.getValue().get()+ ));++ return deviceList.stream()+ .filter(deviceListItem -> !isNullOrEmpty(deviceListItem.getDevEui()))+ .filter(deviceListItem -> duplicateAssetCheck(deviceListItem.getDevEui()))+ .filter(deviceListItem -> !isNullOrEmpty(deviceListItem.getDeviceProfileId()))+ .filter(deviceListItem -> profileMap.containsKey(deviceListItem.getDeviceProfileId()))+ .filter(deviceListItem -> !isNullOrEmpty(profileMap.get(deviceListItem.getDeviceProfileId()).getTagsMap().get(CHIRPSTACK_ASSET_TYPE_TAG)))+ .map(deviceListItem -> createAsset(+ deviceListItem.getDevEui(),+ profileMap.get(deviceListItem.getDeviceProfileId()).getTagsMap().get(CHIRPSTACK_ASSET_TYPE_TAG),+ deviceListItem,+ profileMap.get(deviceListItem.getDeviceProfileId())))+ .filter(Optional::isPresent)+ .map(Optional::get)+ .map(AssetTreeNode::new)+ .toArray(AssetTreeNode[]::new);++ } catch (StatusRuntimeException e) {+ Status status = e.getStatus();+ String errorDetail = String.format(+ "{code=%s, description=%s}",+ status.getCode(),+ status.getDescription() != null ? status.getDescription() : "-"+ );+ LOG.log(+ Level.WARNING,+ String.format("Auto-discovery failed because of a gRPC connection error %s for: %s", errorDetail, getGRPCClientUri()),+ e+ );+ throw e;+ } finally {+ if (channel != null) {+ channel.shutdown();+ try {+ if (!channel.awaitTermination(5, TimeUnit.SECONDS)) {+ channel.shutdownNow();+ }+ } catch (InterruptedException ex) {+ channel.shutdownNow();+ Thread.currentThread().interrupt();+ }+ }+ }+ }++ private int getPort() {+ return getAgent().getPort().orElse(isSecureGRPC() ? 443 : 80);+ }++ private boolean isSecureGRPC() {+ return getAgent().getSecureGRPC().orElse(false);+ }++ private String getGRPCClientUri() {+ return "chirpstack-grpc://" + getAgent().getHost().orElse("-") + ":" + getPort()+ + "/?clientId=" + getAgent().getClientId().orElse("-");+ }++ private ManagedChannel createChannel(String host, int port, boolean isSecure) {+ ManagedChannelBuilder<?> builder = ManagedChannelBuilder.forAddress(host, port);+ if (isSecure) {+ builder.useTransportSecurity();+ } else {+ builder.usePlaintext();+ }+ return builder.build();+ }++ private Optional<Asset<?>> createAsset(String devEUI, String assetTypeName, DeviceListItem deviceItem, DeviceProfile deviceProfile) {+ if (isNullOrEmpty(devEUI) || isNullOrEmpty(assetTypeName) || deviceItem == null || deviceProfile == null) {+ return Optional.empty();+ }++ DeviceRecord record = new DeviceRecord();+ record.setDevEUI(devEUI);+ record.setAssetTypeName(assetTypeName);+ Optional.of(deviceItem.getName())+ .filter(name -> !name.isEmpty())+ .ifPresent(record::setName);++ return Optional.of(assetTypeName)+ .flatMap(name -> resolveAssetClass(name, deviceItem, deviceProfile))+ .flatMap(clazz -> instantiateAsset(clazz, deviceItem, deviceProfile))+ .flatMap(asset -> configureAsset(asset, record));+ }++ private Optional<Class<? extends Asset<?>>> resolveAssetClass(String simpleClassName, DeviceListItem deviceListItem, DeviceProfile deviceProfile) {+ if (isNullOrEmpty(simpleClassName) || deviceListItem == null || deviceProfile == null ) {+ return Optional.empty();+ }++ return ValueUtil.getAssetClass(simpleClassName)+ .or(() -> {+ LOG.warning("Auto-discovery skipped device because of unknown asset type '" + simpleClassName + "', " + chirpStackDeviceToString(deviceListItem, deviceProfile) + " for: " + getGRPCClientUri());+ return Optional.empty();+ });+ }++ private String createDeviceName(DeviceListItem deviceListItem) {+ return Optional.ofNullable(deviceListItem)+ .map(d -> Optional.of(d.getName())+ .filter(name -> !isNullOrEmpty(name))+ .orElseGet(() -> Optional.of(d.getDevEui())+ .filter(devEui -> !isNullOrEmpty(devEui))+ .orElse("")))+ .orElse("");+ }++ private Optional<Asset<?>> instantiateAsset(Class<?> clazz, DeviceListItem deviceListItem, DeviceProfile deviceProfile) {+ if (clazz == null || deviceListItem == null || deviceProfile == null) {+ return Optional.empty();+ }++ Asset<?> asset = null;+ try {+ Constructor<?> constructor = clazz.getConstructor(String.class);+ asset = (Asset<?>) constructor.newInstance(createDeviceName(deviceListItem));+ } catch (ReflectiveOperationException e) {+ LOG.log(Level.WARNING, "Auto-discovery failed to create asset " + chirpStackDeviceToString(deviceListItem, deviceProfile) + " for: " + getGRPCClientUri(), e);+ }+ return Optional.ofNullable(asset);+ }++ private List<DeviceListItem> requestDeviceList(String applicationId, ManagedChannel channel, Metadata headers) {+ if (isNullOrEmpty(applicationId)) {+ return new ArrayList<>();+ }++ DeviceServiceGrpc.DeviceServiceBlockingStub deviceStub = DeviceServiceGrpc.newBlockingStub(channel);+ deviceStub = deviceStub.withInterceptors(MetadataUtils.newAttachHeadersInterceptor(headers));++ int offset = 0;+ List<DeviceListItem> allDevices = new ArrayList<>();++ while (true) {+ ListDevicesRequest request = ListDevicesRequest.newBuilder()+ .setApplicationId(applicationId)+ .setLimit(GRPC_LIST_DEVICE_LIMIT)+ .setOffset(offset)+ .build();+ ListDevicesResponse response = deviceStub+ .withDeadlineAfter(GRPC_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)+ .list(request);++ allDevices.addAll(response.getResultList());++ if (response.getResultList().size() < GRPC_LIST_DEVICE_LIMIT) {+ break;+ }++ offset += GRPC_LIST_DEVICE_LIMIT;+ }++ return allDevices;+ }++ private Optional<DeviceProfile> requestDeviceProfile(String profileId, ManagedChannel channel, Metadata headers) {+ if (isNullOrEmpty(profileId)) {+ return Optional.empty();+ }++ DeviceProfileServiceGrpc.DeviceProfileServiceBlockingStub profileStub = DeviceProfileServiceGrpc.newBlockingStub(channel);+ profileStub = profileStub.withInterceptors(MetadataUtils.newAttachHeadersInterceptor(headers));++ GetDeviceProfileRequest request = GetDeviceProfileRequest.newBuilder()+ .setId(profileId)+ .build();++ DeviceProfile deviceProfile = null;+ try {+ GetDeviceProfileResponse response = profileStub+ .withDeadlineAfter(GRPC_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)+ .get(request);+ deviceProfile = response.getDeviceProfile();+ } catch (io.grpc.StatusRuntimeException e) {+ if (e.getStatus().getCode() != Status.Code.NOT_FOUND) {+ throw e;+ }+ }++ return Optional.ofNullable(deviceProfile);+ }++ private String chirpStackDeviceToString(DeviceListItem deviceListItem, DeviceProfile deviceProfile) {+ return Optional.ofNullable(deviceListItem)+ .map(d ->+ "ChirpStack device{" ++ "devEUI='" + d.getDevEui() + '\'' ++ ", name='" + d.getName() + '\'' ++ ", profile=" + chirpStackProfileToString(deviceProfile) ++ "}"+ )+ .orElse("");+ }++ private String chirpStackProfileToString(DeviceProfile deviceProfile) {+ return Optional.ofNullable(deviceProfile)+ .map(dp ->+ "{" ++ "id='" + dp.getId() + '\'' ++ ", name='" + dp.getName() + '\'' ++ ", tag["+ CHIRPSTACK_ASSET_TYPE_TAG + "] ='" + Optional.ofNullable(deviceProfile.getTagsMap().get(CHIRPSTACK_ASSET_TYPE_TAG)).orElse("") + '\'' ++ "}"+ )+ .orElse("");+ }}diff --git a/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/ConnectedState.java b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/ConnectedState.javanew file mode 100644index 000000000..194c035d5--- /dev/null+++ b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/ConnectedState.java@@ -0,0 +1,62 @@+/*+ * Copyright 2025, OpenRemote Inc.+ *+ * See the CONTRIBUTORS.txt file in the distribution for a+ * full listing of individual contributors.+ *+ * This program is free software: you can redistribute it and/or modify+ * it under the terms of the GNU Affero General Public License as+ * published by the Free Software Foundation, either version 3 of the+ * License, or (at your option) any later version.+ *+ * This program is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+ * GNU Affero General Public License for more details.+ *+ * You should have received a copy of the GNU Affero General Public License+ * along with this program. If not, see <http://www.gnu.org/licenses/>.+ */+package org.openremote.agent.protocol.lorawan.tts;++import org.openremote.model.Container;+import org.openremote.model.asset.agent.ConnectionStatus;++public class ConnectedState implements ConnectionState {+ private final ConnectionStateManager stateManager;+ private final TheThingsStackProtocol protocol;++ public ConnectedState(ConnectionStateManager manager) {+ this.stateManager = manager;+ this.protocol = manager.getProtocol();+ }++ @Override+ public void start(Container container) throws Exception {++ }++ @Override+ public void stop(Container container) {+ protocol.stopSync();+ protocol.stopMqtt(container);+ stateManager.setState(stateManager.getStopState());+ }++ @Override+ public void onMqttConnectionStatusChanged(ConnectionStatus connectionStatus) {+ if (ConnectionStatus.CONNECTED != connectionStatus) {+ protocol.stopSync();+ stateManager.setState(stateManager.getConnectingMqttState());+ stateManager.onMqttConnectionStatusChanged(connectionStatus);+ }+ }++ @Override+ public void onGrpcConnectionStatusChanged(ConnectionStatus connectionStatus) {+ if (ConnectionStatus.CONNECTED != connectionStatus) {+ stateManager.setState(stateManager.getConnectingGrpcState());+ stateManager.onGrpcConnectionStatusChanged(connectionStatus);+ }+ }+}diff --git a/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/ConnectingGrpcState.java b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/ConnectingGrpcState.javanew file mode 100644index 000000000..d2ea3c4c6--- /dev/null+++ b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/ConnectingGrpcState.java@@ -0,0 +1,65 @@+/*+ * Copyright 2025, OpenRemote Inc.+ *+ * See the CONTRIBUTORS.txt file in the distribution for a+ * full listing of individual contributors.+ *+ * This program is free software: you can redistribute it and/or modify+ * it under the terms of the GNU Affero General Public License as+ * published by the Free Software Foundation, either version 3 of the+ * License, or (at your option) any later version.+ *+ * This program is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+ * GNU Affero General Public License for more details.+ *+ * You should have received a copy of the GNU Affero General Public License+ * along with this program. If not, see <http://www.gnu.org/licenses/>.+ */+package org.openremote.agent.protocol.lorawan.tts;++import org.openremote.model.Container;+import org.openremote.model.asset.agent.ConnectionStatus;++public class ConnectingGrpcState implements ConnectionState {+ private final ConnectionStateManager stateManager;+ private final TheThingsStackProtocol protocol;++ public ConnectingGrpcState(ConnectionStateManager manager) {+ this.stateManager = manager;+ this.protocol = manager.getProtocol();+ }++ @Override+ public void start(Container container) throws Exception {++ }++ @Override+ public void stop(Container container) {+ protocol.stopSync();+ protocol.stopMqtt(container);+ stateManager.setState(stateManager.getStopState());+ }++ @Override+ public void onMqttConnectionStatusChanged(ConnectionStatus connectionStatus) {+ if (ConnectionStatus.CONNECTED != connectionStatus) {+ protocol.stopSync();+ stateManager.setState(stateManager.getConnectingMqttState());+ stateManager.onMqttConnectionStatusChanged(connectionStatus);+ }+ }++ @Override+ public void onGrpcConnectionStatusChanged(ConnectionStatus connectionStatus) {+ protocol.setConnectionStatus(connectionStatus);++ if (connectionStatus == ConnectionStatus.CONNECTED) {+ stateManager.setState(stateManager.getConnectedState());+ } else if (connectionStatus == ConnectionStatus.ERROR) {+ stateManager.setState(stateManager.getErrorState());+ }+ }+}diff --git a/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/ConnectingMqttState.java b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/ConnectingMqttState.javanew file mode 100644index 000000000..70893aa67--- /dev/null+++ b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/ConnectingMqttState.java@@ -0,0 +1,63 @@+/*+ * Copyright 2025, OpenRemote Inc.+ *+ * See the CONTRIBUTORS.txt file in the distribution for a+ * full listing of individual contributors.+ *+ * This program is free software: you can redistribute it and/or modify+ * it under the terms of the GNU Affero General Public License as+ * published by the Free Software Foundation, either version 3 of the+ * License, or (at your option) any later version.+ *+ * This program is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+ * GNU Affero General Public License for more details.+ *+ * You should have received a copy of the GNU Affero General Public License+ * along with this program. If not, see <http://www.gnu.org/licenses/>.+ */+package org.openremote.agent.protocol.lorawan.tts;++import org.openremote.model.Container;+import org.openremote.model.asset.agent.ConnectionStatus;++public class ConnectingMqttState implements ConnectionState {+ private final ConnectionStateManager stateManager;+ private final TheThingsStackProtocol protocol;++ public ConnectingMqttState(ConnectionStateManager manager) {+ this.stateManager = manager;+ this.protocol = manager.getProtocol();+ }++ @Override+ public void start(Container container) throws Exception {++ }++ @Override+ public void stop(Container container) {+ protocol.stopMqtt(container);+ stateManager.setState(stateManager.getStopState());+ }++ @Override+ public void onMqttConnectionStatusChanged(ConnectionStatus connectionStatus) {+ if (ConnectionStatus.CONNECTED != connectionStatus) {+ protocol.setConnectionStatus(connectionStatus);+ }++ if (ConnectionStatus.CONNECTED == connectionStatus) {+ protocol.startSync();+ stateManager.setState(stateManager.getConnectingGrpcState());+ } else if (ConnectionStatus.ERROR == connectionStatus) {+ stateManager.setState(stateManager.getErrorState());+ }+ }++ @Override+ public void onGrpcConnectionStatusChanged(ConnectionStatus connectionStatus) {++ }+}diff --git a/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/ConnectionState.java b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/ConnectionState.javanew file mode 100644index 000000000..aa3945bc4--- /dev/null+++ b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/ConnectionState.java@@ -0,0 +1,30 @@+/*+ * Copyright 2025, OpenRemote Inc.+ *+ * See the CONTRIBUTORS.txt file in the distribution for a+ * full listing of individual contributors.+ *+ * This program is free software: you can redistribute it and/or modify+ * it under the terms of the GNU Affero General Public License as+ * published by the Free Software Foundation, either version 3 of the+ * License, or (at your option) any later version.+ *+ * This program is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+ * GNU Affero General Public License for more details.+ *+ * You should have received a copy of the GNU Affero General Public License+ * along with this program. If not, see <http://www.gnu.org/licenses/>.+ */+package org.openremote.agent.protocol.lorawan.tts;++import org.openremote.model.Container;+import org.openremote.model.asset.agent.ConnectionStatus;++public interface ConnectionState {+ void start(Container container) throws Exception;+ void stop(Container container);+ void onMqttConnectionStatusChanged(ConnectionStatus connectionStatus);+ void onGrpcConnectionStatusChanged(ConnectionStatus connectionStatus);+}diff --git a/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/ConnectionStateManager.java b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/ConnectionStateManager.javanew file mode 100644index 000000000..db1b47a22--- /dev/null+++ b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/ConnectionStateManager.java@@ -0,0 +1,103 @@+/*+ * Copyright 2025, OpenRemote Inc.+ *+ * See the CONTRIBUTORS.txt file in the distribution for a+ * full listing of individual contributors.+ *+ * This program is free software: you can redistribute it and/or modify+ * it under the terms of the GNU Affero General Public License as+ * published by the Free Software Foundation, either version 3 of the+ * License, or (at your option) any later version.+ *+ * This program is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+ * GNU Affero General Public License for more details.+ *+ * You should have received a copy of the GNU Affero General Public License+ * along with this program. If not, see <http://www.gnu.org/licenses/>.+ */+package org.openremote.agent.protocol.lorawan.tts;++import org.openremote.model.Container;+import org.openremote.model.asset.agent.ConnectionStatus;++public class ConnectionStateManager implements ConnectionState {+ private final TheThingsStackProtocol protocol;+ private final StartState startState;+ private final StopState stopState;+ private final ConnectingGrpcState connectingGrpcState;+ private final ConnectingMqttState connectingMqttState;+ private final ConnectedState connectedState;+ private final ErrorState errorState;+ private volatile ConnectionState state;++ public ConnectionStateManager(TheThingsStackProtocol protocol) {+ this.protocol = protocol;++ this.startState = new StartState(this);+ this.stopState = new StopState(this);+ this.connectingGrpcState = new ConnectingGrpcState(this);+ this.connectingMqttState = new ConnectingMqttState(this);+ this.connectedState = new ConnectedState(this);+ this.errorState = new ErrorState(this);++ this.state = this.startState;+ }++ @Override+ public synchronized void start(Container container) throws Exception {+ state.start(container);+ }++ @Override+ public synchronized void stop(Container container) {+ state.stop(container);+ }++ @Override+ public synchronized void onMqttConnectionStatusChanged(ConnectionStatus connectionStatus) {+ state.onMqttConnectionStatusChanged(connectionStatus);+ }++ @Override+ public synchronized void onGrpcConnectionStatusChanged(ConnectionStatus connectionStatus) {+ state.onGrpcConnectionStatusChanged(connectionStatus);+ }++ public synchronized void init() {+ this.state = getStartState();+ }++ StartState getStartState() {+ return startState;+ }++ StopState getStopState() {+ return stopState;+ }++ ConnectingGrpcState getConnectingGrpcState() {+ return connectingGrpcState;+ }++ ConnectingMqttState getConnectingMqttState() {+ return connectingMqttState;+ }++ ConnectedState getConnectedState() {+ return connectedState;+ }++ ErrorState getErrorState() {+ return errorState;+ }++ void setState(ConnectionState state) {+ this.state = state;+ }++ TheThingsStackProtocol getProtocol() {+ return this.protocol;+ }+}diff --git a/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/ErrorState.java b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/ErrorState.javanew file mode 100644index 000000000..bddd10169--- /dev/null+++ b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/ErrorState.java@@ -0,0 +1,52 @@+/*+ * Copyright 2025, OpenRemote Inc.+ *+ * See the CONTRIBUTORS.txt file in the distribution for a+ * full listing of individual contributors.+ *+ * This program is free software: you can redistribute it and/or modify+ * it under the terms of the GNU Affero General Public License as+ * published by the Free Software Foundation, either version 3 of the+ * License, or (at your option) any later version.+ *+ * This program is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+ * GNU Affero General Public License for more details.+ *+ * You should have received a copy of the GNU Affero General Public License+ * along with this program. If not, see <http://www.gnu.org/licenses/>.+ */+package org.openremote.agent.protocol.lorawan.tts;++import org.openremote.model.Container;+import org.openremote.model.asset.agent.ConnectionStatus;++public class ErrorState implements ConnectionState {+ private final TheThingsStackProtocol protocol;++ public ErrorState(ConnectionStateManager manager) {+ this.protocol = manager.getProtocol();+ }++ @Override+ public void start(Container container) throws Exception {++ }++ @Override+ public void stop(Container container) {+ protocol.stopSync();+ protocol.stopMqtt(container);+ }++ @Override+ public void onMqttConnectionStatusChanged(ConnectionStatus connectionStatus) {+ protocol.setConnectionStatus(connectionStatus);+ }++ @Override+ public void onGrpcConnectionStatusChanged(ConnectionStatus connectionStatus) {++ }+}diff --git a/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/StartState.java b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/StartState.javanew file mode 100644index 000000000..b34cdb46c--- /dev/null+++ b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/StartState.java@@ -0,0 +1,60 @@+/*+ * Copyright 2025, OpenRemote Inc.+ *+ * See the CONTRIBUTORS.txt file in the distribution for a+ * full listing of individual contributors.+ *+ * This program is free software: you can redistribute it and/or modify+ * it under the terms of the GNU Affero General Public License as+ * published by the Free Software Foundation, either version 3 of the+ * License, or (at your option) any later version.+ *+ * This program is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+ * GNU Affero General Public License for more details.+ *+ * You should have received a copy of the GNU Affero General Public License+ * along with this program. If not, see <http://www.gnu.org/licenses/>.+ */+package org.openremote.agent.protocol.lorawan.tts;++import org.openremote.model.Container;+import org.openremote.model.asset.agent.ConnectionStatus;++public class StartState implements ConnectionState {+ private final ConnectionStateManager stateManager;+ private final TheThingsStackProtocol protocol;++ public StartState(ConnectionStateManager manager) {+ this.stateManager = manager;+ this.protocol = manager.getProtocol();+ }++ @Override+ public void start(Container container) throws Exception {+ try {+ protocol.startMqtt(container);+ stateManager.setState(stateManager.getConnectingMqttState());+ } catch (Exception e) {+ protocol.setConnectionStatus(ConnectionStatus.ERROR);+ stateManager.setState(stateManager.getErrorState());+ throw e;+ }+ }++ @Override+ public void stop(Container container) {++ }++ @Override+ public void onMqttConnectionStatusChanged(ConnectionStatus connectionStatus) {++ }++ @Override+ public void onGrpcConnectionStatusChanged(ConnectionStatus connectionStatus) {++ }+}diff --git a/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/StopState.java b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/StopState.javanew file mode 100644index 000000000..c80e08563--- /dev/null+++ b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/StopState.java@@ -0,0 +1,50 @@+/*+ * Copyright 2025, OpenRemote Inc.+ *+ * See the CONTRIBUTORS.txt file in the distribution for a+ * full listing of individual contributors.+ *+ * This program is free software: you can redistribute it and/or modify+ * it under the terms of the GNU Affero General Public License as+ * published by the Free Software Foundation, either version 3 of the+ * License, or (at your option) any later version.+ *+ * This program is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+ * GNU Affero General Public License for more details.+ *+ * You should have received a copy of the GNU Affero General Public License+ * along with this program. If not, see <http://www.gnu.org/licenses/>.+ */+package org.openremote.agent.protocol.lorawan.tts;++import org.openremote.model.Container;+import org.openremote.model.asset.agent.ConnectionStatus;++public class StopState implements ConnectionState {+ private final TheThingsStackProtocol protocol;++ public StopState(ConnectionStateManager manager) {+ this.protocol = manager.getProtocol();+ }++ @Override+ public void start(Container container) throws Exception {+ }++ @Override+ public void stop(Container container) {++ }++ @Override+ public void onMqttConnectionStatusChanged(ConnectionStatus connectionStatus) {+ protocol.setConnectionStatus(connectionStatus);+ }++ @Override+ public void onGrpcConnectionStatusChanged(ConnectionStatus connectionStatus) {++ }+}diff --git a/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/TheThingsStackAgent.java b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/TheThingsStackAgent.javaindex 7dab287d2..ecead3f9b 100644--- a/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/TheThingsStackAgent.java+++ b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/TheThingsStackAgent.java@@ -24,6 +24,7 @@ import org.openremote.agent.protocol.lorawan.LoRaWANAgent;import org.openremote.agent.protocol.mqtt.MQTTAgentLink;import org.openremote.model.asset.agent.Agent;import org.openremote.model.asset.agent.AgentDescriptor;+import org.openremote.model.auth.UsernamePassword;import org.openremote.model.value.AttributeDescriptor;import org.openremote.model.value.ValueType;@@ -34,8 +35,9 @@ public class TheThingsStackAgent extends LoRaWANAgent<TheThingsStackAgent, TheThpublic static final AttributeDescriptor<String> HOST = Agent.HOST.withOptional(false);public static final AttributeDescriptor<Integer> PORT = Agent.PORT.withOptional(false);- public static final AttributeDescriptor<String> TENANT_ID = new AttributeDescriptor<>("tenantId", ValueType.TEXT);- public static final AttributeDescriptor<String> API_KEY = new AttributeDescriptor<>("apiKey", ValueType.TEXT);+ public static final AttributeDescriptor<UsernamePassword> USERNAME_AND_PASSWORD = Agent.USERNAME_AND_PASSWORD.withOptional(false);+ public static final AttributeDescriptor<String> TENANT_ID = new AttributeDescriptor<>("tenantId", ValueType.TEXT).withOptional(false);+ public static final AttributeDescriptor<Boolean> SECURE_GRPC = new AttributeDescriptor<>("secureGRPC", ValueType.BOOLEAN).withOptional(false);public static final AgentDescriptor<TheThingsStackAgent, TheThingsStackProtocol, MQTTAgentLink> DESCRIPTOR = new AgentDescriptor<>(TheThingsStackAgent.class, TheThingsStackProtocol.class, MQTTAgentLink.class@@ -65,12 +67,12 @@ public class TheThingsStackAgent extends LoRaWANAgent<TheThingsStackAgent, TheThreturn this;}- public Optional<String> getApiKey() {- return getAttributes().getValue(API_KEY);+ public Optional<Boolean> getSecureGRPC() {+ return getAttributes().getValue(SECURE_GRPC);}- public TheThingsStackAgent setApiKey(String apiKey) {- getAttributes().getOrCreate(API_KEY).setValue(apiKey);+ public TheThingsStackAgent setSecureGRPC(Boolean secureGRPC) {+ getAttributes().getOrCreate(SECURE_GRPC).setValue(secureGRPC);return this;}}diff --git a/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/TheThingsStackProtocol.java b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/TheThingsStackProtocol.javaindex 7567f0efc..8d479c886 100644--- a/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/TheThingsStackProtocol.java+++ b/agent/src/main/java/org/openremote/agent/protocol/lorawan/tts/TheThingsStackProtocol.java@@ -20,54 +20,107 @@package org.openremote.agent.protocol.lorawan.tts;import com.fasterxml.jackson.core.JsonProcessingException;-import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;+import com.google.protobuf.FieldMask;+import io.grpc.ManagedChannel;+import io.grpc.ManagedChannelBuilder;+import io.grpc.Metadata;+import io.grpc.Status;+import io.grpc.StatusRuntimeException;+import io.grpc.stub.MetadataUtils;import org.openremote.agent.protocol.lorawan.AbstractLoRaWANProtocol;-import org.openremote.agent.protocol.lorawan.CsvRecord;+import org.openremote.agent.protocol.lorawan.DeviceRecord;+import org.openremote.agent.protocol.lorawan.GrpcRetryingRunner;+import org.openremote.agent.protocol.lorawan.GrpcRetryingRunner.StreamBreakSignalException;+import org.openremote.agent.protocol.lorawan.LoRaWANMQTTProtocol;+import org.openremote.agent.protocol.lorawan.UniqueBlockingQueue;+import org.openremote.agent.protocol.mqtt.MQTTAgent;import org.openremote.agent.protocol.mqtt.MQTTAgentLink;+import org.openremote.agent.protocol.mqtt.MQTTProtocol;+import org.openremote.model.Container;+import org.openremote.model.asset.Asset;+import org.openremote.model.asset.agent.ConnectionStatus;import org.openremote.model.attribute.Attribute;import org.openremote.model.attribute.AttributeEvent;import org.openremote.model.query.filter.NumberPredicate;import org.openremote.model.syslog.SyslogCategory;+import org.openremote.model.util.UniqueIdentifierGenerator;+import org.openremote.model.util.ValueUtil;import org.openremote.model.value.AttributeDescriptor;import org.openremote.model.value.JsonPathFilter;import org.openremote.model.value.ValueFilter;+import ttn.lorawan.v3.EndDeviceOuterClass;+import ttn.lorawan.v3.EndDeviceRegistryGrpc;+import ttn.lorawan.v3.EventsGrpc;+import ttn.lorawan.v3.EventsOuterClass;+import ttn.lorawan.v3.Identifiers;-import java.net.URI;-import java.net.http.HttpClient;-import java.net.http.HttpRequest;-import java.net.http.HttpResponse;+import java.lang.reflect.Constructor;+import java.time.Duration;import java.util.AbstractMap;import java.util.ArrayList;import java.util.Collections;import java.util.HashMap;+import java.util.Iterator;import java.util.List;import java.util.Map;import java.util.Optional;+import java.util.Set;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.CopyOnWriteArraySet;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicReference;+import java.util.concurrent.locks.Lock;+import java.util.concurrent.locks.ReentrantLock;import java.util.logging.Level;import java.util.logging.Logger;+import java.util.stream.Collectors;-import static org.openremote.agent.protocol.lorawan.tts.TheThingsStackAgent.API_KEY;-import static org.openremote.agent.protocol.lorawan.tts.TheThingsStackAgent.HOST;-import static org.openremote.agent.protocol.lorawan.tts.TheThingsStackAgent.TENANT_ID;+import static org.openremote.agent.protocol.lorawan.LoRaWANConstants.ASSET_TYPE_TAG;+import static org.openremote.agent.protocol.lorawan.tts.TheThingsStackAgent.*;import static org.openremote.model.syslog.SyslogCategory.PROTOCOL;-import static org.openremote.model.util.MapAccess.getBoolean;+import static org.openremote.model.util.TextUtil.isNullOrEmpty;public class TheThingsStackProtocol extends AbstractLoRaWANProtocol<TheThingsStackProtocol, TheThingsStackAgent> {public static final Logger LOG = SyslogCategory.getLogger(PROTOCOL, TheThingsStackProtocol.class);+ public static final String THE_THINGS_STACK_ASSET_TYPE_TAG = ASSET_TYPE_TAG;public static final String PROTOCOL_DISPLAY_NAME = "The Things Stack";- public static final int GET_DEVICE_LIMIT = 1000;+ public static final int GRPC_LIST_DEVICE_LIMIT = 1000;+ public static final long GRPC_TIMEOUT_MILLIS = 10000L;+ public static final long GRPC_STREAM_RUNNER_INITIAL_BACKOFF_MILLIS = 5000L;+ public static final long GRPC_STREAM_RUNNER_MAX_BACKOFF_MILLIS = 5*60000L;+ /*+ The bulkSyncRunner can be activated to perform periodic full discovery+ of LoRaWAN devices.- private Optional<Map<String, String>> devEuiToDeviceIdMap = Optional.empty();+ public static final long GRPC_BULK_SYNC_RUNNER_INITIAL_BACKOFF_MILLIS = 5000L;+ public static final long GRPC_BULK_SYNC_RUNNER_MAX_BACKOFF_MILLIS = 5*60000L;+ public static final long GRPC_BULK_SYNC_RUNNER_CONTINUATION_SCHEDULE_MILLIS = 12*60*60000L;+ */+ public static final long GRPC_BULK_SYNC_RUNNER_KEEP_ALIVE_MILLIS = 10000L;+ public static final long GRPC_BULK_SYNC_RUNNER_KEEP_ALIVE_TIMEOUT_MILLIS = 4000L;+ public static final long GRPC_SINGLE_DEVICE_SYNC_RUNNER_INITIAL_BACKOFF_MILLIS = 100L;+ public static final long GRPC_SINGLE_DEVICE_SYNC_RUNNER_MAX_BACKOFF_MILLIS = 5*60000L;+ public static final String BULK_SYNC_RUNNER_NAME = "Bulk Sync Runner";+ public static final String SINGLE_DEVICE_SYNC_RUNNER_NAME = "Single Device Sync Runner";+ public static final String DEVICE_EVENT_STREAMER_NAME = "Device Event Streamer";++ private final ConnectionStateManager connectionStateManager = new ConnectionStateManager(this);+ private final AtomicReference<Map<String, EndDeviceOuterClass.EndDevice>> ttsDeviceMap = new AtomicReference<>(new ConcurrentHashMap<>());+ private final UniqueBlockingQueue<String> deviceIdQueue = new UniqueBlockingQueue<>();+ private final Set<String> ignoreDevEuiSet = new CopyOnWriteArraySet<>();++ private final GrpcRetryingRunner bulkSyncRunner = new GrpcRetryingRunner(BULK_SYNC_RUNNER_NAME, this::getGRPCClientUri);+ private final GrpcRetryingRunner deviceEventStreamer = new GrpcRetryingRunner(DEVICE_EVENT_STREAMER_NAME, this::getGRPCClientUri);+ private volatile GrpcRetryingRunner singleDeviceSyncRunner;+ private final Lock singleDeviceSyncLock = new ReentrantLock();public TheThingsStackProtocol(TheThingsStackAgent agent) {super(agent);}- public static final String THE_THINGS_STACK_TEST = "THE_THINGS_STACK_TEST";-@Overridepublic String getProtocolName() {return PROTOCOL_DISPLAY_NAME;@@ -75,18 +128,70 @@ public class TheThingsStackProtocol extends AbstractLoRaWANProtocol<TheThingsSta@Overridepublic String getProtocolInstanceUri() {- return "tts://" + getAgent().getHost().orElse("-") + ":"- + getAgent().getPort().map(p -> p.toString()).orElse("-");+ return "tts-mqtt://" + getAgent().getMqttHost().orElse("-") + ":" + getAgent().getMqttPort().map(Object::toString).orElse("-")+ + "/?clientId=" + getAgent().getClientId().orElse("-");+ }++ @Override+ public void start(org.openremote.model.Container container) throws Exception {+ super.start(container);++ connectionStateManager.init();+ connectionStateManager.start(container);+ }++ @Override+ public void stop(org.openremote.model.Container container) throws Exception {+ connectionStateManager.stop(container);+ }++ @Override+ protected void doUnlinkDevice(String devEui) {+ ignoreDevEuiSet.remove(devEui);}@Overridepublic boolean onAgentAttributeChanged(AttributeEvent event) {- if (TENANT_ID.getName().equals(event.getName())) {+ if (TENANT_ID.getName().equals(event.getName()) ||+ API_KEY.getName().equals(event.getName()) ||+ SECURE_GRPC.getName().equals(event.getName()))+ {return true;}+return super.onAgentAttributeChanged(event);}+ @Override+ protected MQTTProtocol createMqttClientProtocol(MQTTAgent agent) {+ return new LoRaWANMQTTProtocol(agent) {+ @Override+ public String getProtocolName() {+ return "The Things Stack MQTT Client";+ }++ @Override+ protected void onConnectionStatusChanged(ConnectionStatus connectionStatus) {+ onMqttConnectionStatusChanged(connectionStatus);+ }+ };+ }++ protected void onMqttConnectionStatusChanged(ConnectionStatus connectionStatus) {+ connectionStateManager.onMqttConnectionStatusChanged(connectionStatus);+ }++ protected void onGrpcConnectionStatusChanged(ConnectionStatus connectionStatus) {+ connectionStateManager.onGrpcConnectionStatusChanged(connectionStatus);+ }++ protected void setConnectionStatus(ConnectionStatus connectionStatus) {+ LoRaWANMQTTProtocol mqttProtocol = (LoRaWANMQTTProtocol)getMqttProtocol();+ if (mqttProtocol != null) {+ mqttProtocol.setStatus(connectionStatus);+ }+ }+@Overrideprotected boolean checkCSVImportPrerequisites() {boolean isOk = super.checkCSVImportPrerequisites();@@ -99,22 +204,19 @@ public class TheThingsStackProtocol extends AbstractLoRaWANProtocol<TheThingsStafor (AbstractMap.SimpleEntry<AttributeDescriptor<String>, Optional<String>> item : list) {if (!item.getValue().map(attrValue -> !attrValue.trim().isEmpty()).orElse(false)) {isOk = false;- LOG.log(Level.WARNING, "CSV import failed because agent attribute '" + item.getKey().getName() + "' is missing");+ LOG.log(Level.WARNING, "CSV import failed because agent attribute '" + item.getKey().getName() + "' is missing for: " + getProtocolInstanceUri());}}- initializeDevEuiToDeviceIdMap();- if(devEuiToDeviceIdMap.isEmpty()) {- isOk = false;- }+ initDeviceMap();return isOk;}@Overrideprotected List<String> createWildcardSubscriptionTopicList() {- Optional<String> tenantId = getAgent().getTenantId().map(id -> id.trim());- Optional<String> applicationId = getAgent().getApplicationId().map(id -> id.trim());+ Optional<String> tenantId = getAgent().getTenantId().map(String::trim);+ Optional<String> applicationId = getAgent().getApplicationId().map(String::trim);return applicationId.map(id -> Collections.singletonList("v3/" + (tenantId.isPresent() ? (applicationId.get() + "@" + tenantId.get()) : applicationId.get()) + "/devices/+/up"))@@ -122,58 +224,53 @@ public class TheThingsStackProtocol extends AbstractLoRaWANProtocol<TheThingsSta}@Override- protected boolean configureMQTTSubscriptionTopic(Attribute<?> attribute, MQTTAgentLink agentLink, CsvRecord csvRecord) {- if (attribute == null || agentLink == null || csvRecord == null) {+ protected boolean configureMQTTSubscriptionTopic(Attribute<?> attribute, MQTTAgentLink agentLink, DeviceRecord deviceRecord) {+ if (attribute == null || agentLink == null || deviceRecord == null) {return false;}boolean isOk = true;- Optional<String> applicationId = getAgent().getApplicationId().map(id -> id.trim());- Optional<String> apiKey = getAgent().getApiKey().map(key -> key.trim());- Optional<String> tenantId = getAgent().getTenantId().map(id -> id.trim());- Optional<String> devEUI = Optional.ofNullable(csvRecord.getDevEUI()).map(eui -> eui.trim());+ Optional<String> applicationId = getAgent().getApplicationId().map(String::trim);+ Optional<String> apiKey = getAgent().getApiKey().map(String::trim);+ Optional<String> tenantId = getAgent().getTenantId().map(String::trim);+ Optional<String> devEUI = Optional.ofNullable(deviceRecord.getDevEUI()).map(String::trim).map(String::toUpperCase);if (applicationId.isPresent() && devEUI.isPresent() && apiKey.isPresent()) {- if (devEuiToDeviceIdMap.isEmpty()) {- return false;- }- Optional<String> deviceId = devEuiToDeviceIdMap.flatMap(map -> Optional.ofNullable(map.get(devEUI.get())));- if (deviceId.isPresent()) {- agentLink.setSubscriptionTopic("v3/" + (tenantId.isPresent() ? (applicationId.get() + "@" + tenantId.get()) : applicationId.get()) + "/devices/" + deviceId.get() + "/up");+ EndDeviceOuterClass.EndDevice device = ttsDeviceMap.get().get(devEUI.get());+ if (device != null) {+ String deviceId = device.getIds().getDeviceId();+ agentLink.setSubscriptionTopic("v3/" + (tenantId.isPresent() ? (applicationId.get() + "@" + tenantId.get()) : applicationId.get()) + "/devices/" + deviceId + "/up");} else {- LOG.log(- Level.WARNING,- "CSV import failure because couldn't find device on LoRaWAN network server: " + csvRecord+ LOG.warning(+ "CSV import failure because couldn't find device " + deviceRecord + " on LoRaWAN network server for: " + getProtocolInstanceUri());isOk = false;}}+return isOk;}@Override- protected boolean configureMQTTPublishTopic(Attribute<?> attribute, MQTTAgentLink agentLink, CsvRecord csvRecord) {- if (attribute == null || agentLink == null || csvRecord == null) {+ protected boolean configureMQTTPublishTopic(Attribute<?> attribute, MQTTAgentLink agentLink, DeviceRecord deviceRecord) {+ if (attribute == null || agentLink == null || deviceRecord == null) {return false;}boolean isOk = true;- Optional<String> applicationId = getAgent().getApplicationId().map(id -> id.trim());- Optional<String> apiKey = getAgent().getApiKey().map(key -> key.trim());- Optional<String> tenantId = getAgent().getTenantId().map(id -> id.trim());- Optional<String> devEUI = Optional.ofNullable(csvRecord.getDevEUI()).map(eui -> eui.trim());+ Optional<String> applicationId = getAgent().getApplicationId().map(String::trim);+ Optional<String> apiKey = getAgent().getApiKey().map(String::trim);+ Optional<String> tenantId = getAgent().getTenantId().map(String::trim);+ Optional<String> devEUI = Optional.ofNullable(deviceRecord.getDevEUI()).map(String::trim).map(String::toUpperCase);if (applicationId.isPresent() && devEUI.isPresent() && apiKey.isPresent()) {- if (devEuiToDeviceIdMap.isEmpty()) {- return false;- }- Optional<String> deviceId = devEuiToDeviceIdMap.flatMap(map -> Optional.ofNullable(map.get(devEUI.get())));- if (deviceId.isPresent()) {- agentLink.setPublishTopic("v3/" + (tenantId.isPresent() ? (applicationId.get() + "@" + tenantId.get()) : applicationId.get()) + "/devices/" + deviceId.get() + "/down/push");+ EndDeviceOuterClass.EndDevice device = ttsDeviceMap.get().get(devEUI.get());+ if (device != null) {+ String deviceId = device.getIds().getDeviceId();+ agentLink.setPublishTopic("v3/" + (tenantId.isPresent() ? (applicationId.get() + "@" + tenantId.get()) : applicationId.get()) + "/devices/" + deviceId + "/down/push");} else {- LOG.log(- Level.WARNING,- "CSV import failure because couldn't find device on the LoRaWAN network server: " + csvRecord+ LOG.warning(+ "CSV import failure because couldn't find device " + deviceRecord + " on LoRaWAN network server for: " + getProtocolInstanceUri());isOk = false;}@@ -183,8 +280,8 @@ public class TheThingsStackProtocol extends AbstractLoRaWANProtocol<TheThingsSta}@Override- protected boolean configureMQTTMessageMatchFilterAndPredicate(Attribute<?> attribute, MQTTAgentLink agentLink, CsvRecord csvRecord) {- if (attribute == null || agentLink == null || csvRecord == null) {+ protected boolean configureMQTTMessageMatchFilterAndPredicate(Attribute<?> attribute, MQTTAgentLink agentLink, DeviceRecord deviceRecord) {+ if (attribute == null || agentLink == null || deviceRecord == null) {return false;}getAgentConfigUplinkPort(attribute).ifPresent(port ->@@ -195,8 +292,8 @@ public class TheThingsStackProtocol extends AbstractLoRaWANProtocol<TheThingsSta}@Override- protected boolean configureMQTTWriteValueTemplate(Attribute<?> attribute, MQTTAgentLink agentLink, CsvRecord csvRecord) {- if (attribute == null || agentLink == null || csvRecord == null) {+ protected boolean configureMQTTWriteValueTemplate(Attribute<?> attribute, MQTTAgentLink agentLink, DeviceRecord deviceRecord) {+ if (attribute == null || agentLink == null || deviceRecord == null) {return false;}@@ -219,15 +316,97 @@ public class TheThingsStackProtocol extends AbstractLoRaWANProtocol<TheThingsStaagentLink.setWriteValue(json);} catch (JsonProcessingException e) {isOk = false;- LOG.log(Level.SEVERE, "CSV import failure for CSV record: " + csvRecord, e);+ LOG.log(Level.SEVERE, "CSV import failure " + deviceRecord + " for: " + getProtocolInstanceUri(), e);}}return isOk;}- private void initializeDevEuiToDeviceIdMap() {+ @Override+ protected String generateAssetId(String devEui) {+ return UniqueIdentifierGenerator.generateId("TTS_" + getAgent().getId() + devEui);+ }++ protected String getGRPCClientUri() {+ return "tts-grpc://" + getAgent().getHost().orElse("-") + ":" + getPort()+ + "/?clientId=" + getAgent().getClientId().orElse("-");+ }++ protected void startMqtt(Container container) throws Exception {+ startMqttProtocol(container);+ }++ protected void stopMqtt(Container container) {+ stopMqttProtocol(container);+ }++ protected void startSync() {+ Optional<String> host = getAgent().getHost().map(String::trim);+ int port = getPort();+ boolean isSecureGRPC = isSecureGRPC();+ Optional<String> apiKey = getAgent().getApiKey().map(String::trim);+ Optional<String> applicationId = getAgent().getApplicationId().map(String::trim);++ List<String> missingFields = new ArrayList<>();+ if (host.isEmpty()) {+ missingFields.add(HOST.getName());+ }+ if (applicationId.isEmpty()) {+ missingFields.add(APPLICATION_ID.getName());+ }+ if (apiKey.isEmpty()) {+ missingFields.add(API_KEY.getName());+ }++ if (!missingFields.isEmpty()) {+ LOG.warning("Auto-discovery skipped due to missing agent attributes {" + String.join(", ", missingFields) + "} for: " + getProtocolInstanceUri());+ onGrpcConnectionStatusChanged(ConnectionStatus.ERROR);+ return;+ }++ /*+ The bulkSyncRunner can be activated to perform periodic full discovery+ of LoRaWAN devices.++ bulkSyncRunner.start(+ () -> createChannel(host.get(), port, isSecureGRPC, false),+ this::performBulkDeviceSync,+ true,+ Duration.ofMillis(GRPC_BULK_SYNC_RUNNER_INITIAL_BACKOFF_MILLIS),+ Duration.ofMillis(GRPC_BULK_SYNC_RUNNER_MAX_BACKOFF_MILLIS),+ Duration.ofMillis(GRPC_BULK_SYNC_RUNNER_CONTINUATION_SCHEDULE_MILLIS)+ );+ */++ deviceEventStreamer.startStream(+ () -> createChannel(host.get(), port, isSecureGRPC, true),+ this::performDevicesUpdate,+ Duration.ofMillis(GRPC_STREAM_RUNNER_INITIAL_BACKOFF_MILLIS),+ Duration.ofMillis(GRPC_STREAM_RUNNER_MAX_BACKOFF_MILLIS),+ this::onGrpcConnectionStatusChanged+ );+ }++ protected void stopSync() {+ bulkSyncRunner.shutdown();+ deviceEventStreamer.shutdown();++ singleDeviceSyncLock.lock();+ try {+ if (singleDeviceSyncRunner != null) {+ singleDeviceSyncRunner.shutdown();+ singleDeviceSyncRunner = null;+ deviceIdQueue.clear();+ }+ } finally {+ singleDeviceSyncLock.unlock();+ }+ }++ private void startSingleDeviceSync() {Optional<String> host = getAgent().getHost().map(String::trim);- Optional<Integer> port = getAgent().getPort();+ int port = getPort();+ boolean isSecureGRPC = isSecureGRPC();Optional<String> apiKey = getAgent().getApiKey().map(String::trim);Optional<String> applicationId = getAgent().getApplicationId().map(String::trim);@@ -235,79 +414,649 @@ public class TheThingsStackProtocol extends AbstractLoRaWANProtocol<TheThingsStareturn;}- Map<String, String> map = new HashMap<>();+ singleDeviceSyncLock.lock();+ try {+ if (deviceIdQueue.size() > 0 && singleDeviceSyncRunner == null) {+ singleDeviceSyncRunner = new GrpcRetryingRunner(SINGLE_DEVICE_SYNC_RUNNER_NAME, this::getGRPCClientUri);+ singleDeviceSyncRunner.start(+ () -> createChannel(host.get(), port, isSecureGRPC, false),+ this::performSingleDeviceSync,+ false,+ Duration.ofMillis(GRPC_SINGLE_DEVICE_SYNC_RUNNER_INITIAL_BACKOFF_MILLIS),+ Duration.ofMillis(GRPC_SINGLE_DEVICE_SYNC_RUNNER_MAX_BACKOFF_MILLIS)+ );+ }+ } finally {+ singleDeviceSyncLock.unlock();+ }+ }++ private int getPort() {+ return getAgent().getPort().orElse(isSecureGRPC() ? 443 : 80);+ }++ private boolean isSecureGRPC() {+ return getAgent().getSecureGRPC().orElse(true);+ }++ private ManagedChannel createChannel(String host, int port, boolean isSecure, boolean withKeepAlive) {+ ManagedChannelBuilder<?> builder = ManagedChannelBuilder.forAddress(host, port);+ if (isSecure) {+ builder.useTransportSecurity();+ } else {+ builder.usePlaintext();+ }++ if (withKeepAlive) {+ builder+ .keepAliveTime(GRPC_BULK_SYNC_RUNNER_KEEP_ALIVE_MILLIS, TimeUnit.MILLISECONDS)+ .keepAliveTimeout(GRPC_BULK_SYNC_RUNNER_KEEP_ALIVE_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)+ .keepAliveWithoutCalls(true);+ }++ return builder.build();+ }++ private void performDevicesUpdate(ManagedChannel channel) throws Exception {+ Optional<String> apiKey = getAgent().getApiKey().map(String::trim);+ Optional<String> applicationId = getAgent().getApplicationId().map(String::trim);++ if (apiKey.isEmpty() || applicationId.isEmpty()) {+ throw new IllegalStateException("Missing API key or application ID");+ }++ Metadata headers = new Metadata();+ Metadata.Key<String> AUTHORIZATION_HEADER =+ Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER);+ headers.put(AUTHORIZATION_HEADER, "Bearer " + apiKey.get());++ EventsGrpc.EventsBlockingStub stub = EventsGrpc.newBlockingStub(channel)+ .withInterceptors(MetadataUtils.newAttachHeadersInterceptor(headers));+++ EventsOuterClass.StreamEventsRequest request = EventsOuterClass.StreamEventsRequest.newBuilder()+ .addIdentifiers(+ Identifiers.EntityIdentifiers.newBuilder()+ .setApplicationIds(+ Identifiers.ApplicationIdentifiers.newBuilder()+ .setApplicationId(applicationId.get())+ .build()+ )+ )+ .build();++ boolean isStreamEstablished = false;+ boolean isStatusUpdate = true;++ Iterator<EventsOuterClass.Event> events = stub.stream(request);try {- String hostWithPort = port- .map(p -> String.format("%s:%d", host.get(), p))- .orElse(host.get());+ while (events.hasNext()) {+ if (Thread.currentThread().isInterrupted()) {+ throw new InterruptedException("[" + DEVICE_EVENT_STREAMER_NAME + "] interrupted" );+ }- HttpClient client = HttpClient.newHttpClient();- ObjectMapper mapper = new ObjectMapper();+ EventsOuterClass.Event event = events.next();- int page = 1;+ if (isStatusUpdate) {+ isStatusUpdate = false;+ onGrpcConnectionStatusChanged(ConnectionStatus.CONNECTED);+ }+ isStreamEstablished = true;- while (true) {- String url = String.format(- "%s://%s/api/v3/applications/%s/devices?field_mask=name,ids&limit=%d&page=%d",- getTTSApiScheme(), hostWithPort, applicationId.get(), GET_DEVICE_LIMIT, page+ String eventName = event.getName();+ String typeUrl = event.getData().getTypeUrl();++ LOG.finest(() ->+ "[" + DEVICE_EVENT_STREAMER_NAME + "] Received event (name=" + eventName ++ ", typeUrl=" + typeUrl + ") for: " + getProtocolInstanceUri());- HttpRequest request = HttpRequest.newBuilder()- .uri(URI.create(url))- .header("Authorization", "Bearer " + apiKey.get())- .header("Accept", "application/json")- .GET()- .build();+ if (eventName.equals("end_device.batch.delete")) {+ try {+ Identifiers.EndDeviceIdentifiersList list;- HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());+ if (event.getData().is(Identifiers.EndDeviceIdentifiersList.class)) {+ list = event.getData().unpack(Identifiers.EndDeviceIdentifiersList.class);+ } else {+ LOG.warning(() ->+ "[" + DEVICE_EVENT_STREAMER_NAME + "] Bad batch delete event. Unexpected payload type (name=" + eventName ++ ", typeUrl=" + typeUrl + ") for: " + getProtocolInstanceUri()+ );+ continue;+ }- if (response.statusCode() != 200) {- LOG.log(- Level.WARNING,- "CSV import failure because couldn't retrieve device list from LoRaWAN server [HTTP " +- response.statusCode() + "]: " + response.body()- );- break;+ for (Identifiers.EndDeviceIdentifiers devIds : list.getEndDeviceIdsList()) {+ if (isNullOrEmpty(devIds.getDeviceId())) {+ continue;+ }++ List<String> devEuiList = new ArrayList<>();+ ttsDeviceMap.get().entrySet().removeIf(entry -> {+ boolean isRemove = devIds.getDeviceId().equals(entry.getValue().getIds().getDeviceId());+ if (isRemove) {+ String devEui = bytesToHex(devIds.getDevEui().toByteArray()).toUpperCase();+ if (!isNullOrEmpty(devEui)) {+ devEuiList.add(devEui);+ }+ }+ return isRemove;+ });++ for (String devEui : devEuiList) {+ ignoreDevEuiSet.remove(devEui);+ }++ LOG.finest(() ->+ "[" + DEVICE_EVENT_STREAMER_NAME + "] Deleted device (deviceId=" + devIds.getDeviceId() ++ ") from batch event (eventName=" + eventName + ") for: " + getProtocolInstanceUri()+ );+ }+ } catch (Exception ex) {+ LOG.log(+ Level.WARNING,+ "[" + DEVICE_EVENT_STREAMER_NAME + "] Failed to unpack event (eventName=" + eventName ++ ", typeUrl=" + typeUrl + ") for: " + getProtocolInstanceUri(), ex+ );+ }++ continue;}- JsonNode root = mapper.readTree(response.body());- JsonNode devices = root.path("end_devices");+ Identifiers.EndDeviceIdentifiers devIds = extractDeviceIds(event);+ String deviceId = (devIds != null ? devIds.getDeviceId() : null);+ String devEui = (devIds != null ? bytesToHex(devIds.getDevEui().toByteArray()).toUpperCase() : null);+ if (isNullOrEmpty(deviceId)) {+ continue;+ }++ switch (eventName) {+ case "as.up.data.forward":+ case "js.join.accept": {+ if (!isNullOrEmpty(devEui)) {+ if (!ttsDeviceMap.get().containsKey(devEui) || (!devEuiSet.contains(devEui) && !ignoreDevEuiSet.contains(devEui))) {+ LOG.finest(() ->+ "[" + DEVICE_EVENT_STREAMER_NAME + "] Scheduling sync for device (eventName=" + eventName ++ ", devEui=" + devEui + ", deviceId=" + deviceId + ") for: " + getProtocolInstanceUri()+ );+ queueDeviceIdForSync(deviceId);+ }+ }+ break;+ }++ case "end_device.create":+ case "end_device.update": {+ LOG.finest(() ->+ "[" + DEVICE_EVENT_STREAMER_NAME + "] Scheduling sync for device (eventName=" + eventName ++ ", deviceId=" + deviceId + ") for: " + getProtocolInstanceUri()+ );+ queueDeviceIdForSync(deviceId);+ break;+ }- if (!devices.isArray() || devices.isEmpty()) {- break;+ case "end_device.delete": {+ String devEuiToRemove = ttsDeviceMap.get().entrySet().stream()+ .filter(entry -> deviceId.equals(entry.getValue().getIds().getDeviceId()))+ .map(Map.Entry::getKey)+ .findFirst()+ .orElse(null);+ if (devEuiToRemove != null) {+ LOG.finest(() ->+ "[" + DEVICE_EVENT_STREAMER_NAME + "] Deleting device (eventName=" + eventName ++ ", devEui=" + devEuiToRemove + ", deviceId=" + deviceId + ") for: " + getProtocolInstanceUri()+ );+ ignoreDevEuiSet.remove(devEuiToRemove);+ }+ ttsDeviceMap.get().entrySet().removeIf(entry -> deviceId.equals(entry.getValue().getIds().getDeviceId()));+ break;+ }}+ }+ } catch (StatusRuntimeException ex) {+ if (!GrpcRetryingRunner.isNonRetryableCode(ex) && isStreamEstablished) {+ throw new StreamBreakSignalException("[" + DEVICE_EVENT_STREAMER_NAME + "] ended unexpectedly." , ex);+ } else {+ throw ex;+ }+ }++ throw new StreamBreakSignalException("[" + DEVICE_EVENT_STREAMER_NAME + "] ended normally.", null);+ }++ private void performBulkDeviceSync(ManagedChannel channel) throws Exception {+ Optional<String> apiKey = getAgent().getApiKey().map(String::trim);+ Optional<String> applicationId = getAgent().getApplicationId().map(String::trim);++ if (apiKey.isEmpty() || applicationId.isEmpty()) {+ throw new IllegalStateException("Missing API key or application ID");+ }++ Metadata headers = new Metadata();+ Metadata.Key<String> AUTHORIZATION_HEADER =+ Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER);+ headers.put(AUTHORIZATION_HEADER, "Bearer " + apiKey.get());++ EndDeviceRegistryGrpc.EndDeviceRegistryBlockingStub stub = EndDeviceRegistryGrpc+ .newBlockingStub(channel)+ .withInterceptors(MetadataUtils.newAttachHeadersInterceptor(headers))+ .withDeadlineAfter(GRPC_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);+- for (JsonNode device : devices) {- JsonNode ids = device.path("ids");- String devEui = ids.path("dev_eui").asText();- String deviceId = ids.path("device_id").asText();+ Map<String, EndDeviceOuterClass.EndDevice> map = new HashMap<>();+ int page = 1;+ boolean hasMore = true;- if (!devEui.isEmpty()) {- map.put(devEui.toLowerCase(), deviceId);+ while (hasMore) {+ if (Thread.currentThread().isInterrupted()) {+ throw new InterruptedException("[" + BULK_SYNC_RUNNER_NAME + "] interrupted" );+ }++ EndDeviceOuterClass.ListEndDevicesRequest request = EndDeviceOuterClass.ListEndDevicesRequest.newBuilder()+ .setApplicationIds(+ Identifiers.ApplicationIdentifiers.newBuilder()+ .setApplicationId(applicationId.get())+ .build()+ )+ .setFieldMask(+ FieldMask.newBuilder()+ .addPaths("name")+ .addPaths("ids")+ .addPaths("attributes")+ .build()+ )+ .setLimit(GRPC_LIST_DEVICE_LIMIT)+ .setPage(page)+ .build();++ EndDeviceOuterClass.EndDevices response = stub.list(request);++ response.getEndDevicesList().forEach(device -> {+ if (!isNullOrEmpty(device.getIds().getDeviceId())) {+ String devEui = bytesToHex(device.getIds().getDevEui().toByteArray());+ if (!isNullOrEmpty(devEui)) {+ map.put(devEui.toUpperCase(), device);}}+ });- if (devices.size() < GET_DEVICE_LIMIT) {- break;+ hasMore = response.getEndDevicesCount() == GRPC_LIST_DEVICE_LIMIT;+ page++;+ }++ ignoreDevEuiSet.clear();+ ttsDeviceMap.set(new ConcurrentHashMap<>(map));++ LOG.fine(() -> "[" + BULK_SYNC_RUNNER_NAME + "] Synced TTS device map {size=" + map.size() + "} for URI: " + getProtocolInstanceUri());++ mergeDiscoveredDevices();+ }++ private void performSingleDeviceSync(ManagedChannel channel) throws Exception {++ String deviceId = deviceIdQueue.peek();++ if (isNullOrEmpty(deviceId)) {+ return;+ }++ Optional<String> apiKey = getAgent().getApiKey().map(String::trim);+ Optional<String> applicationId = getAgent().getApplicationId().map(String::trim);++ if (apiKey.isEmpty() || applicationId.isEmpty()) {+ throw new IllegalStateException("Missing API key or application ID");+ }++ Metadata headers = new Metadata();+ Metadata.Key<String> AUTHORIZATION_HEADER =+ Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER);+ headers.put(AUTHORIZATION_HEADER, "Bearer " + apiKey.get());++ EndDeviceRegistryGrpc.EndDeviceRegistryBlockingStub stub = EndDeviceRegistryGrpc+ .newBlockingStub(channel)+ .withInterceptors(MetadataUtils.newAttachHeadersInterceptor(headers))+ .withDeadlineAfter(GRPC_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);++ Identifiers.EndDeviceIdentifiers deviceIds = Identifiers.EndDeviceIdentifiers.newBuilder()+ .setDeviceId(deviceId)+ .setApplicationIds(+ Identifiers.ApplicationIdentifiers.newBuilder()+ .setApplicationId(applicationId.get())+ .build()+ )+ .build();++ EndDeviceOuterClass.GetEndDeviceRequest request =+ EndDeviceOuterClass.GetEndDeviceRequest.newBuilder()+ .setEndDeviceIds(deviceIds)+ .setFieldMask(FieldMask.newBuilder()+ .addPaths("name")+ .addPaths("ids")+ .addPaths("attributes")+ .build()+ )+ .build();++ try {+ EndDeviceOuterClass.EndDevice device = stub.get(request);++ if (device != null && !isNullOrEmpty(device.getIds().getDeviceId())) {+ String devEui = bytesToHex(device.getIds().getDevEui().toByteArray()).toUpperCase();+ if (!isNullOrEmpty(devEui)) {+ ttsDeviceMap.get().put(devEui, device);++ LOG.finest(() ->+ "[" + SINGLE_DEVICE_SYNC_RUNNER_NAME + "] Synced device (devEui=" + devEui ++ ", deviceId=" + device.getIds().getDeviceId() + ") for: " + getProtocolInstanceUri()+ );++ mergeDiscoveredDevice(devEui, device);}+ }+ } catch (StatusRuntimeException ex) {+ if (ex.getStatus().getCode() != Status.Code.NOT_FOUND) {+ Status status = ex.getStatus();+ String errorDetail = String.format(+ "{code=%s, description=%s}",+ status.getCode(),+ status.getDescription() != null ? status.getDescription() : "-"+ );+ LOG.log(+ Level.WARNING,+ String.format(+ "[" + SINGLE_DEVICE_SYNC_RUNNER_NAME + "] Device sync (deviceId=%s) failed because of a gRPC connection error %s for: %s",+ deviceId, errorDetail, getGRPCClientUri()+ ),+ ex+ );+ throw ex;+ }+ }++ boolean acquired = false;+ try {+ singleDeviceSyncLock.lockInterruptibly();+ acquired = true;++ if (Thread.currentThread().isInterrupted()) {+ throw new InterruptedException("[" + SINGLE_DEVICE_SYNC_RUNNER_NAME + "] interrupted" );+ }++ deviceIdQueue.take();+ singleDeviceSyncRunner = null;+ startSingleDeviceSync();+ } finally {+ if (acquired) {+ singleDeviceSyncLock.unlock();+ }+ }+ }++ private void queueDeviceIdForSync(String deviceId) throws InterruptedException {+ if (!isNullOrEmpty(deviceId)) {+ boolean acquired = false;+ try {+ singleDeviceSyncLock.lockInterruptibly();+ acquired = true;++ if (Thread.currentThread().isInterrupted()) {+ throw new InterruptedException("[" + DEVICE_EVENT_STREAMER_NAME + "] interrupted" );+ }++ deviceIdQueue.put(deviceId);+ startSingleDeviceSync();++ } finally {+ if (acquired) {+ singleDeviceSyncLock.unlock();+ }+ }+ }+ }++ private Optional<Asset<?>> createAsset(String devEUI, String assetTypeName, EndDeviceOuterClass.EndDevice device) {+ if (isNullOrEmpty(devEUI) || isNullOrEmpty(assetTypeName) || device == null) {+ return Optional.empty();+ }++ DeviceRecord record = new DeviceRecord();+ record.setDevEUI(devEUI);+ record.setAssetTypeName(assetTypeName);+ Optional.of(device.getName())+ .filter(name -> !name.isEmpty())+ .ifPresent(record::setName);++ return Optional.of(assetTypeName)+ .flatMap(name -> resolveAssetClass(name, device))+ .flatMap(clazz -> instantiateAsset(clazz, device))+ .flatMap(asset -> configureAsset(asset, record));+ }++ private Optional<Class<? extends Asset<?>>> resolveAssetClass(String simpleClassName, EndDeviceOuterClass.EndDevice device) {+ if (isNullOrEmpty(simpleClassName) || device == null) {+ return Optional.empty();+ }++ return ValueUtil.getAssetClass(simpleClassName)+ .or(() -> {+ LOG.warning("Auto-discovery skipped device because of unknown asset type: '" + simpleClassName + "', " + ttsDeviceToString(device) + " for: " + getProtocolInstanceUri());+ return Optional.empty();+ });+ }++ private Optional<Asset<?>> instantiateAsset(Class<?> clazz, EndDeviceOuterClass.EndDevice device) {+ if (clazz == null || device == null) {+ return Optional.empty();+ }+ Asset<?> asset = null;+ try {+ Constructor<?> constructor = clazz.getConstructor(String.class);+ asset = (Asset<?>) constructor.newInstance(createDeviceName(device));+ } catch (ReflectiveOperationException e) {+ LOG.log(Level.WARNING, "Auto-discovery failed to create asset '" + ttsDeviceToString(device) + "' for: " + getProtocolInstanceUri(), e);+ }+ return Optional.ofNullable(asset);+ }++ private Map<String, Asset<?>> createAssetsFromDiscoveredDevices(Map<String, EndDeviceOuterClass.EndDevice> discoveredDeviceMap) {+ return discoveredDeviceMap+ .entrySet()+ .stream()+ .filter(entry -> duplicateAssetCheck(entry.getKey()))+ .filter(entry -> !isNullOrEmpty(entry.getValue().getAttributesMap().get(THE_THINGS_STACK_ASSET_TYPE_TAG)))+ .map(entry -> {+ String devEui = entry.getKey();+ Optional<Asset<?>> assetOptional = createAsset(+ devEui,+ entry.getValue().getAttributesMap().get(THE_THINGS_STACK_ASSET_TYPE_TAG),+ entry.getValue());+ return assetOptional.map(asset -> new AbstractMap.SimpleEntry<>(devEui, asset));+ })+ .filter(Optional::isPresent)+ .map(Optional::get)+ .collect(Collectors.toMap(+ Map.Entry::getKey,+ Map.Entry::getValue+ ));+ }++ private void mergeDiscoveredDevices() throws InterruptedException {+ Map<String, EndDeviceOuterClass.EndDevice> deviceMap = ttsDeviceMap.get();+ Map<String, Asset<?>> assetMap = createAssetsFromDiscoveredDevices(deviceMap);++ for (Map.Entry<String, EndDeviceOuterClass.EndDevice> entry : deviceMap.entrySet()) {++ if (Thread.currentThread().isInterrupted()) {+ throw new InterruptedException();+ }++ if (assetMap.containsKey(entry.getKey())) {+ ignoreDevEuiSet.remove(entry.getKey());++ try {+ assetService.mergeAsset(assetMap.get(entry.getKey()));+ } catch (Exception e) {+ LOG.log(Level.WARNING, "Asset merge failed '" + assetMap.get(entry.getKey()) + "' for: " + getProtocolInstanceUri(), e);+ }+ } else {+ ignoreDevEuiSet.add(entry.getKey());+ }+ }+ }++ private void mergeDiscoveredDevice(String devEui, EndDeviceOuterClass.EndDevice device) {+ Map<String, EndDeviceOuterClass.EndDevice> deviceMap = Collections.singletonMap(devEui, device);+ Map<String, Asset<?>> assetMap = createAssetsFromDiscoveredDevices(deviceMap);++ if (assetMap.size() == 1) {+ ignoreDevEuiSet.remove(devEui);++ try {+ assetService.mergeAsset(assetMap.get(devEui));+ } catch (Exception e) {+ LOG.log(Level.WARNING, "Asset merge failed '" + assetMap.get(devEui) + "' for: " + getProtocolInstanceUri(), e);+ }+ } else {+ ignoreDevEuiSet.add(devEui);+ }+ }++ private String createDeviceName(EndDeviceOuterClass.EndDevice device) {+ return Optional.ofNullable(device)+ .map(d -> Optional.of(d.getName())+ .filter(name -> !isNullOrEmpty(name))+ .orElseGet(() -> Optional.of(d.getIds())+ .map(ids -> bytesToHex(ids.getDevEui().toByteArray()))+ .filter(devEui -> !isNullOrEmpty(devEui))+ .orElse("")))+ .orElse("");+ }++ private Identifiers.EndDeviceIdentifiers extractDeviceIds(EventsOuterClass.Event event) {+ for (Identifiers.EntityIdentifiers id : event.getIdentifiersList()) {+ if (id.hasDeviceIds()) {+ return id.getDeviceIds();+ }+ }+ return null;+ }++ private static String bytesToHex(byte[] bytes) {+ if (bytes == null) {+ return "";+ }+ StringBuilder sb = new StringBuilder();+ for (byte b : bytes) {+ sb.append(String.format("%02X", b));+ }+ return sb.toString();+ }++ private String ttsDeviceToString(EndDeviceOuterClass.EndDevice device) {+ return Optional.ofNullable(device)+ .map(d ->+ "TTS device{" ++ "devEUI='" + bytesToHex(d.getIds().getDevEui().toByteArray()) + '\'' ++ ", deviceId='" + d.getIds().getDeviceId() + '\'' ++ ", name='" + d.getName() + '\'' ++ "}"+ )+ .orElse("");+ }++ private void initDeviceMap() {+ Optional<String> host = getAgent().getHost().map(String::trim);+ int port = getPort();+ boolean isSecureGRPC = isSecureGRPC();+ Optional<String> apiKey = getAgent().getApiKey().map(String::trim);+ Optional<String> applicationId = getAgent().getApplicationId().map(String::trim);++ if (host.isEmpty() || apiKey.isEmpty() || applicationId.isEmpty()) {+ return;+ }++ ManagedChannel channel = createChannel(host.get(), port, isSecureGRPC, false);++ Metadata headers = new Metadata();+ Metadata.Key<String> AUTHORIZATION_HEADER = Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER);+ headers.put(AUTHORIZATION_HEADER, "Bearer " + apiKey.get());++ try {+ EndDeviceRegistryGrpc.EndDeviceRegistryBlockingStub stub = EndDeviceRegistryGrpc+ .newBlockingStub(channel)+ .withInterceptors(MetadataUtils.newAttachHeadersInterceptor(headers))+ .withDeadlineAfter(GRPC_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);++ Map<String, EndDeviceOuterClass.EndDevice> map = new HashMap<>();+ int page = 1;+ boolean hasMore = true;++ while (hasMore) {+ if (Thread.currentThread().isInterrupted()) {+ return;+ }++ EndDeviceOuterClass.ListEndDevicesRequest request = EndDeviceOuterClass.ListEndDevicesRequest.newBuilder()+ .setApplicationIds(+ Identifiers.ApplicationIdentifiers.newBuilder()+ .setApplicationId(applicationId.get())+ .build()+ )+ .setFieldMask(+ FieldMask.newBuilder()+ .addPaths("name")+ .addPaths("ids")+ .addPaths("attributes")+ .build()+ )+ .setLimit(GRPC_LIST_DEVICE_LIMIT)+ .setPage(page)+ .build();++ EndDeviceOuterClass.EndDevices response = stub.list(request);++ response.getEndDevicesList().forEach(device -> {+ if (!isNullOrEmpty(device.getIds().getDeviceId())) {+ String devEui = bytesToHex(device.getIds().getDevEui().toByteArray());+ if (!isNullOrEmpty(devEui)) {+ map.put(devEui.toUpperCase(), device);+ }+ }+ });++ hasMore = response.getEndDevicesCount() == GRPC_LIST_DEVICE_LIMIT;page++;}- this.devEuiToDeviceIdMap = Optional.of(map);- } catch (Exception e) {+ ignoreDevEuiSet.clear();+ ttsDeviceMap.set(new ConcurrentHashMap<>(map));++ } catch (StatusRuntimeException e) {+ Status status = e.getStatus();+ String errorDetail = String.format(+ "{code=%s, description=%s}",+ status.getCode(),+ status.getDescription() != null ? status.getDescription() : "-"+ );LOG.log(Level.WARNING,- "CSV import failure because couldn't retrieve device list from the LoRaWAN server", e+ String.format("CSV import failed because of a gRPC connection error %s for: %s", errorDetail, getProtocolInstanceUri()),+ e);+ throw e;+ } finally {+ if (channel != null) {+ channel.shutdown();+ try {+ if (!channel.awaitTermination(5, TimeUnit.SECONDS)) {+ channel.shutdownNow();+ }+ } catch (InterruptedException ex) {+ channel.shutdownNow();+ Thread.currentThread().interrupt();+ }+ }}}-- private String getTTSApiScheme() {- return Optional.ofNullable(container)- .map(c -> getBoolean(c.getConfig(), THE_THINGS_STACK_TEST, false))- .map(isTest -> isTest ? "http" : "https")- .orElse("https");- }}diff --git a/gradle.properties b/gradle.propertiesindex 0b2fc2ad2..dfe6aaa30 100644--- a/gradle.properties+++ b/gradle.properties@@ -75,3 +75,4 @@ moquetteVersion = 0.18.0corsFilterVersion = 3.1junitVersion = 5.14.1mockitoVersion = 5.21.0+grpcVersion = 1.69.0diff --git a/test/build.gradle b/test/build.gradleindex 3eeb2bade..4539e9182 100644--- a/test/build.gradle+++ b/test/build.gradle@@ -39,6 +39,13 @@ dependencies {testImplementation "com.github.moquette-io:moquette:$moquetteVersion"testImplementation "com.hivemq:hivemq-mqtt-client:$hiveMQClientVersion"++ testImplementation "io.grpc:grpc-netty-shaded:$grpcVersion"+ testImplementation "io.grpc:grpc-protobuf:$grpcVersion"+ testImplementation "io.grpc:grpc-stub:$grpcVersion"++ testImplementation files("../agent/libs/chirpstack-api-4.14.1.jar")+ testImplementation files("../agent/libs/tts-api-3.34.3.jar")}tasks.withType(Test) {diff --git a/test/src/test/groovy/org/openremote/test/protocol/lorawan/chirpstack/ChirpStackTest.groovy b/test/src/test/groovy/org/openremote/test/protocol/lorawan/chirpstack/ChirpStackTest.groovyindex beb34993a..b48f19f01 100644--- a/test/src/test/groovy/org/openremote/test/protocol/lorawan/chirpstack/ChirpStackTest.groovy+++ b/test/src/test/groovy/org/openremote/test/protocol/lorawan/chirpstack/ChirpStackTest.groovy@@ -19,11 +19,22 @@*/package org.openremote.test.protocol.lorawan.chirpstack+import io.chirpstack.api.DeviceListItem+import io.chirpstack.api.DeviceProfile+import io.chirpstack.api.DeviceProfileServiceGrpc+import io.chirpstack.api.DeviceServiceGrpcimport com.hivemq.client.mqtt.MqttClientimport com.hivemq.client.mqtt.mqtt3.Mqtt3AsyncClientimport com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publishimport groovy.json.JsonOutputimport groovy.json.JsonSlurper+import io.chirpstack.api.GetDeviceProfileRequest+import io.chirpstack.api.GetDeviceProfileResponse+import io.chirpstack.api.ListDevicesRequest+import io.chirpstack.api.ListDevicesResponse+import io.grpc.ServerBuilder+import io.grpc.stub.StreamObserver+import io.grpc.Statusimport io.moquette.broker.Serverimport io.moquette.broker.config.MemoryConfigimport org.openremote.agent.protocol.lorawan.chirpstack.ChirpStackAgent@@ -37,15 +48,16 @@ import org.openremote.model.asset.agent.AgentResourceimport org.openremote.model.asset.agent.ConnectionStatusimport org.openremote.model.attribute.AttributeEventimport org.openremote.model.file.FileInfo+import org.openremote.model.query.AssetQueryimport org.openremote.test.ManagerContainerTraitimport spock.lang.Sharedimport spock.lang.Specificationimport spock.util.concurrent.PollingConditions-import java.nio.file.Pathimport java.util.concurrent.CopyOnWriteArrayListimport java.util.concurrent.TimeUnit+import static org.openremote.agent.protocol.lorawan.LoRaWANConstants.ATTRIBUTE_NAME_DEV_EUIimport static org.openremote.container.security.IdentityProvider.OR_ADMIN_PASSWORDimport static org.openremote.container.security.IdentityProvider.OR_ADMIN_PASSWORD_DEFAULTimport static org.openremote.model.util.MapAccess.getString@@ -68,6 +80,8 @@ class ChirpStackTest extends Specification implements ManagerContainerTrait {static final Integer DOWNLINK_PORT = 4static final Double TEMPERATURE_VALUE = 20.4static final Double HUMIDITY_VALUE = 56.4+ static final String PROFILE_ID = "4711"+ static final String API_KEY = "123456789ABCDEFG"@Sharedint mqttBrokerPort@@ -76,22 +90,27 @@ class ChirpStackTest extends Specification implements ManagerContainerTrait {Server mqttBroker@Shared- Path tempDataDir+ int grpcPort++ @Shared+ ChirpStackGrpcServer grpcServer@SharedMqtt3AsyncClient mqttClientdef setupSpec() {- tempDataDir = File.createTempDir('moquette_data', '').toPath()mqttBrokerPort = findEphemeralPort()def props = new Properties()props.setProperty('port', mqttBrokerPort.toString())- // Moquette assumes linux paths so need to convert windows paths- props.setProperty('persistent_store', tempDataDir.resolve('moquette_store.mapdb').toString().replace('\\', '/'))+ props.setProperty('persistence_enabled', 'false')def config = new MemoryConfig(props)mqttBroker = new Server()mqttBroker.startServer(config)+ grpcPort = findEphemeralPort()+ grpcServer = new ChirpStackGrpcServer(grpcPort)+ grpcServer.start()+mqttClient = MqttClient.builder().useMqttVersion3().identifier("testClientId")@@ -104,10 +123,10 @@ class ChirpStackTest extends Specification implements ManagerContainerTrait {def cleanupSpec() {mqttClient?.disconnect()mqttBroker?.stopServer()- tempDataDir?.deleteDir()+ grpcServer?.stop()}- def "ChirpStack Integration Test"() {+ def "ChirpStack CSV Import Test"() {given: "expected conditions"def conditions = new PollingConditions(timeout: 10, delay: 0.2)@@ -159,15 +178,123 @@ class ChirpStackTest extends Specification implements ManagerContainerTrait {AssetTreeNode[] assets = agentResource.doProtocolAssetImport(null, agent.getId(), null, fileInfo)then: "new assets should have been imported"- assert assets != null- assert assets.length == 2- def node1 = assets.find {it.asset.getAttribute(DEV_EUI).map {DEV_EUI_1.equalsIgnoreCase(it.value.get())}.orElse(false)}- def node2 = assets.find {it.asset.getAttribute(DEV_EUI).map {DEV_EUI_2.equalsIgnoreCase(it.value.get())}.orElse(false)}- assert node1 != null- assert node2 != null+ conditions.eventually {+ assert assets != null+ assert assets.length == 2+ def node1 = assets.find { it.asset.getAttribute(DEV_EUI).map { DEV_EUI_1.equalsIgnoreCase(it.value.get()) }.orElse(false) }+ def node2 = assets.find { it.asset.getAttribute(DEV_EUI).map { DEV_EUI_2.equalsIgnoreCase(it.value.get()) }.orElse(false) }+ assert node1 != null+ assert node2 != null+ def asset1 = assetStorageService.find(new AssetQuery().attributeValue(ATTRIBUTE_NAME_DEV_EUI, DEV_EUI_1.toUpperCase()))+ def asset2 = assetStorageService.find(new AssetQuery().attributeValue(ATTRIBUTE_NAME_DEV_EUI, DEV_EUI_2.toUpperCase()))+ assert asset1 != null+ assert asset2 != null+ }++ when: "the device sends a LoRaWAN uplink message"+ def json = [+ fPort: UPLINK_PORT,+ object: [+ Temperature: TEMPERATURE_VALUE,+ Humidity: HUMIDITY_VALUE+ ]+ ]+ mqttClient.publishWith()+ .topic("application/${APPLICATION_ID}/device/${DEV_EUI_1}/event/up")+ .payload(JsonOutput.toJson(json).bytes)+ .send()++ then: "asset attribute values should be updated"+ conditions.eventually {+ def asset = assetStorageService.find(new AssetQuery().attributeValue(ATTRIBUTE_NAME_DEV_EUI, DEV_EUI_1.toUpperCase()))+ assert asset.getAttribute(TEMPERATURE).flatMap {it.value}.map {it == TEMPERATURE_VALUE}.orElse(false)+ assert asset.getAttribute(RELATIVE_HUMIDITY).flatMap {it.value}.map {it == HUMIDITY_VALUE}.orElse(false)+ }++ when: "an asset attribute value is written"+ def asset1 = assetStorageService.find(new AssetQuery().attributeValue(ATTRIBUTE_NAME_DEV_EUI, DEV_EUI_1.toUpperCase()))+ def downlinkMessages = new CopyOnWriteArrayList<String>()+ mqttClient.subscribeWith()+ .topicFilter("application/${APPLICATION_ID}/device/${DEV_EUI_1}/command/down")+ .callback { Mqtt3Publish publish ->+ downlinkMessages.add(new String(publish.payloadAsBytes))+ }.send()+ assetProcessingService.sendAttributeEvent(new AttributeEvent(asset1.id, SWITCH, true))++ then: "a LoRaWAN downlink message should have been published"+ conditions.eventually {+ assert downlinkMessages.size() == 1+ def downlinkMessage = new JsonSlurper().parseText(downlinkMessages.get(0))+ assert downlinkMessage != null+ assert downlinkMessage.devEui == DEV_EUI_1+ assert downlinkMessage.fPort == DOWNLINK_PORT+ assert downlinkMessage.data == "DAE="+ }+ }++ def "ChirpStack Auto Discovery Test"() {+ given: "expected conditions"+ def conditions = new PollingConditions(timeout: 10, delay: 0.2)++ when: "the container starts"+ def container = startContainer(defaultConfig(), defaultServices())+ def assetStorageService = container.getService(AssetStorageService.class)+ def assetProcessingService = container.getService(AssetProcessingService.class)+ def agentService = container.getService(AgentService.class)++ and: "a ChirpStack agent is created"+ def agent = new ChirpStackAgent("ChirpStackAgent")+ agent.setRealm(MASTER_REALM)+ agent.setMqttHost(HOST)+ agent.setMqttPort(mqttBrokerPort)+ agent.setHost(HOST)+ agent.setPort(grpcPort)+ agent.setClientId(CLIENT_ID)+ agent.setApplicationId(APPLICATION_ID)+ agent.setApiKey(API_KEY)+ agent = assetStorageService.merge(agent)++ then: "the protocol instance for the agent should be created"+ conditions.eventually {+ assert agentService.getProtocolInstance(agent.id) != null+ assert ((ChirpStackProtocol)agentService.getProtocolInstance(agent.id)) != null+ }++ and: "the connection status should be CONNECTED"+ conditions.eventually {+ agent = assetStorageService.find(agent.id)+ agent.getAttribute(Agent.STATUS).get().getValue().get() == ConnectionStatus.CONNECTED+ }++ when: "an authenticated admin user"+ def accessToken = authenticate(+ container,+ MASTER_REALM,+ KEYCLOAK_CLIENT_ID,+ MASTER_REALM_ADMIN_USER,+ getString(container.getConfig(), OR_ADMIN_PASSWORD, OR_ADMIN_PASSWORD_DEFAULT)+ ).token++ and: "the agent resource"+ def agentResource = getClientApiTarget(serverUri(serverPort), MASTER_REALM, accessToken).proxy(AgentResource.class)++ AssetTreeNode[] assets = agentResource.doProtocolAssetDiscovery(null, agent.getId(), null)++ then: "new assets should have been discovered"+ conditions.eventually {+ assert assets != null+ assert assets.length == 2+ def node1 = assets.find { it.asset.getAttribute(DEV_EUI).map { DEV_EUI_1.equalsIgnoreCase(it.value.get()) }.orElse(false) }+ def node2 = assets.find { it.asset.getAttribute(DEV_EUI).map { DEV_EUI_2.equalsIgnoreCase(it.value.get()) }.orElse(false) }+ assert node1 != null+ assert node2 != null+ def asset1 = assetStorageService.find(new AssetQuery().attributeValue(ATTRIBUTE_NAME_DEV_EUI, DEV_EUI_1.toUpperCase()))+ def asset2 = assetStorageService.find(new AssetQuery().attributeValue(ATTRIBUTE_NAME_DEV_EUI, DEV_EUI_2.toUpperCase()))+ assert asset1 != null+ assert asset2 != null+ }when: "the device sends a LoRaWAN uplink message"- def asset1 = node1.assetdef json = [fPort: UPLINK_PORT,object: [@@ -182,12 +309,13 @@ class ChirpStackTest extends Specification implements ManagerContainerTrait {then: "asset attribute values should be updated"conditions.eventually {- def asset = assetStorageService.find(asset1.id)+ def asset = assetStorageService.find(new AssetQuery().attributeValue(ATTRIBUTE_NAME_DEV_EUI, DEV_EUI_1.toUpperCase()))assert asset.getAttribute(TEMPERATURE).flatMap {it.value}.map {it == TEMPERATURE_VALUE}.orElse(false)assert asset.getAttribute(RELATIVE_HUMIDITY).flatMap {it.value}.map {it == HUMIDITY_VALUE}.orElse(false)}when: "an asset attribute value is written"+ def asset1 = assetStorageService.find(new AssetQuery().attributeValue(ATTRIBUTE_NAME_DEV_EUI, DEV_EUI_1.toUpperCase()))def downlinkMessages = new CopyOnWriteArrayList<String>()mqttClient.subscribeWith().topicFilter("application/${APPLICATION_ID}/device/${DEV_EUI_1}/command/down")@@ -206,4 +334,78 @@ class ChirpStackTest extends Specification implements ManagerContainerTrait {assert downlinkMessage.data == "DAE="}}++ static class ChirpStackGrpcServer {+ private final int port+ private io.grpc.Server server++ ChirpStackGrpcServer(int port) {+ this.port = port+ }++ void start() {+ server = ServerBuilder.forPort(port)+ .addService(new DeviceService())+ .addService(new DeviceProfileService())+ .build()+ .start()+ }++ void stop() {+ if (server != null) {+ server.shutdownNow()+ }+ }++ private static class DeviceService extends DeviceServiceGrpc.DeviceServiceImplBase {+ @Override+ void list(ListDevicesRequest request, StreamObserver<ListDevicesResponse> responseObserver) {+ DeviceListItem device1 = DeviceListItem.newBuilder()+ .setDevEui(DEV_EUI_1)+ .setName(ASSET_NAME_1)+ .setDeviceProfileId(PROFILE_ID)+ .build()+ DeviceListItem device2 = DeviceListItem.newBuilder()+ .setDevEui(DEV_EUI_2)+ .setName(ASSET_NAME_2)+ .setDeviceProfileId(PROFILE_ID)+ .build()+ ListDevicesResponse response = ListDevicesResponse.newBuilder()+ .addResult(device1)+ .addResult(device2)+ .setTotalCount(2)+ .build()++ responseObserver.onNext(response)+ responseObserver.onCompleted()+ }+ }++ private static class DeviceProfileService extends DeviceProfileServiceGrpc.DeviceProfileServiceImplBase {+ @Override+ void get(GetDeviceProfileRequest request, StreamObserver<GetDeviceProfileResponse> responseObserver) {+ if (PROFILE_ID != request.id) {+ responseObserver.onError(+ Status.NOT_FOUND+ .withDescription("Device profile not found: ${request.id}")+ .asRuntimeException()+ )+ return+ }++ DeviceProfile profile = DeviceProfile.newBuilder()+ .setName("TestProfile")+ .setId(request.getId())+ .putTags(ChirpStackProtocol.CHIRPSTACK_ASSET_TYPE_TAG, "ChirpStackTestAsset")+ .build()++ GetDeviceProfileResponse response = GetDeviceProfileResponse.newBuilder()+ .setDeviceProfile(profile)+ .build()++ responseObserver.onNext(response)+ responseObserver.onCompleted()+ }+ }+ }}diff --git a/test/src/test/groovy/org/openremote/test/protocol/lorawan/tts/TheThingsStackTest.groovy b/test/src/test/groovy/org/openremote/test/protocol/lorawan/tts/TheThingsStackTest.groovyindex 9efe25f90..bc9d6d87e 100644--- a/test/src/test/groovy/org/openremote/test/protocol/lorawan/tts/TheThingsStackTest.groovy+++ b/test/src/test/groovy/org/openremote/test/protocol/lorawan/tts/TheThingsStackTest.groovy@@ -19,14 +19,19 @@*/package org.openremote.test.protocol.lorawan.tts+import com.google.protobuf.Any+import com.google.protobuf.ByteString+import com.google.protobuf.Duration+import com.google.protobuf.Timestampimport com.hivemq.client.mqtt.MqttClientimport com.hivemq.client.mqtt.mqtt3.Mqtt3AsyncClientimport com.hivemq.client.mqtt.mqtt3.message.publish.Mqtt3Publish-import com.sun.net.httpserver.HttpExchange-import com.sun.net.httpserver.HttpHandler-import com.sun.net.httpserver.HttpServerimport groovy.json.JsonOutputimport groovy.json.JsonSlurper+import io.grpc.ServerBuilder+import io.grpc.Status+import io.grpc.StatusRuntimeException+import io.grpc.stub.StreamObserverimport io.moquette.broker.Serverimport io.moquette.broker.config.MemoryConfigimport org.openremote.agent.protocol.lorawan.tts.TheThingsStackAgent@@ -40,22 +45,27 @@ import org.openremote.model.asset.agent.AgentResourceimport org.openremote.model.asset.agent.ConnectionStatusimport org.openremote.model.attribute.AttributeEventimport org.openremote.model.file.FileInfo+import org.openremote.model.query.AssetQueryimport org.openremote.test.ManagerContainerTraitimport spock.lang.Sharedimport spock.lang.Specificationimport spock.util.concurrent.PollingConditions+import ttn.lorawan.v3.*-import java.nio.file.Pathimport java.util.concurrent.CopyOnWriteArrayList-import java.util.concurrent.Executorsimport java.util.concurrent.TimeUnit-import static org.openremote.agent.protocol.lorawan.tts.TheThingsStackProtocol.THE_THINGS_STACK_TEST+import static org.openremote.agent.protocol.lorawan.LoRaWANConstants.ATTRIBUTE_NAME_DEV_EUI+import static org.openremote.agent.protocol.lorawan.tts.TheThingsStackProtocol.THE_THINGS_STACK_ASSET_TYPE_TAGimport static org.openremote.container.security.IdentityProvider.OR_ADMIN_PASSWORDimport static org.openremote.container.security.IdentityProvider.OR_ADMIN_PASSWORD_DEFAULT-import static org.openremote.model.util.MapAccess.getStringimport static org.openremote.model.Constants.*+import static org.openremote.model.util.MapAccess.getStringimport static org.openremote.setup.integration.model.asset.TheThingsStackTestAsset.*+import static ttn.lorawan.v3.EndDeviceOuterClass.*+import static ttn.lorawan.v3.EventsOuterClass.Event+import static ttn.lorawan.v3.EventsOuterClass.StreamEventsRequest+import static ttn.lorawan.v3.Identifiers.*class TheThingsStackTest extends Specification implements ManagerContainerTrait {@@ -78,12 +88,6 @@ class TheThingsStackTest extends Specification implements ManagerContainerTraitstatic final Double TEMPERATURE_VALUE = 20.4static final Double HUMIDITY_VALUE = 56.4- @Shared- int ttsServerPort-- @Shared- HttpServer ttsServer-@Sharedint mqttBrokerPort@@ -91,22 +95,27 @@ class TheThingsStackTest extends Specification implements ManagerContainerTraitServer mqttBroker@Shared- Path tempDataDir+ int grpcPort++ @Shared+ TheThingsStackGrpcServer grpcServer@SharedMqtt3AsyncClient mqttClientdef setupSpec() {- tempDataDir = File.createTempDir('moquette_data', '').toPath()mqttBrokerPort = findEphemeralPort()def props = new Properties()props.setProperty('port', mqttBrokerPort.toString())- // Moquette assumes linux paths so need to convert windows paths- props.setProperty('persistent_store', tempDataDir.resolve('moquette_store.mapdb').toString().replace('\\', '/'))+ props.setProperty('persistence_enabled', 'false')def config = new MemoryConfig(props)mqttBroker = new Server()mqttBroker.startServer(config)+ grpcPort = findEphemeralPort()+ grpcServer = new TheThingsStackGrpcServer(grpcPort)+ grpcServer.start()+mqttClient = MqttClient.builder().useMqttVersion3().identifier("testClientId")@@ -114,56 +123,24 @@ class TheThingsStackTest extends Specification implements ManagerContainerTrait.serverPort(mqttBrokerPort).buildAsync()mqttClient.connect().get(2, TimeUnit.SECONDS)-- ttsServerPort = findEphemeralPort()- ttsServer = HttpServer.create(new InetSocketAddress(ttsServerPort), 0)- ttsServer.createContext("/api/v3/applications/${APPLICATION_ID}/devices", new HttpHandler() {- @Override- void handle(HttpExchange exchange) throws IOException {- String responseJson = """- {- "end_devices": [- {- "ids": {- "dev_eui": "${DEV_EUI_1}",- "device_id": "${DEVICE_ID_1}"- }- },- {- "ids": {- "dev_eui": "${DEV_EUI_2}",- "device_id": "${DEVICE_ID_2}"- }- }- ]- }- """- exchange.responseHeaders.add("Content-Type", "application/json")- exchange.sendResponseHeaders(200, responseJson.getBytes().length)- exchange.responseBody.withCloseable {- it.write(responseJson.getBytes())- }- }- })-- ttsServer.setExecutor(Executors.newSingleThreadExecutor())- ttsServer.start()}def cleanupSpec() {- ttsServer?.stop(0)mqttClient?.disconnect()mqttBroker?.stopServer()- tempDataDir?.deleteDir()+ grpcServer?.stop()+ }++ def cleanup() {+ grpcServer?.shutdownEventStream()}- def "TheThingsStack Integration Test"() {+ def "TheThingsStack CSV Import Test"() {given: "expected conditions"def conditions = new PollingConditions(timeout: 10, delay: 0.2)when: "the container starts"def container = startContainer(defaultConfig(), defaultServices())- container.getConfig().put(THE_THINGS_STACK_TEST, true.toString())def assetStorageService = container.getService(AssetStorageService.class)def assetProcessingService = container.getService(AssetProcessingService.class)def agentService = container.getService(AgentService.class)@@ -172,7 +149,8 @@ class TheThingsStackTest extends Specification implements ManagerContainerTraitdef agent = new TheThingsStackAgent("TheThingsStackAgent")agent.setRealm(MASTER_REALM)agent.setHost(HOST)- agent.setPort(ttsServerPort)+ agent.setPort(grpcPort)+ agent.setSecureGRPC(false)agent.setMqttHost(HOST)agent.setMqttPort(mqttBrokerPort)agent.setClientId(CLIENT_ID)@@ -187,7 +165,16 @@ class TheThingsStackTest extends Specification implements ManagerContainerTraitassert ((TheThingsStackProtocol)agentService.getProtocolInstance(agent.id)) != null}- and: "the connection status should be CONNECTED"+ and: "the gRPC event stream connection should have been established"+ conditions.eventually {+ assert grpcServer.getEventObserverCount() == 1+ }++ when: "the initial gRPC event has been received"+ def streamStartEvent = TheThingsStackGrpcServer.createStreamStartEvent(APPLICATION_ID)+ grpcServer.pushEvent(streamStartEvent)++ then: "the connection status should be CONNECTED"conditions.eventually {agent = assetStorageService.find(agent.id)agent.getAttribute(Agent.STATUS).get().getValue().get() == ConnectionStatus.CONNECTED@@ -214,15 +201,20 @@ class TheThingsStackTest extends Specification implements ManagerContainerTraitAssetTreeNode[] assets = agentResource.doProtocolAssetImport(null, agent.getId(), null, fileInfo)then: "new assets should have been imported"- assert assets != null- assert assets.length == 2- def node1 = assets.find {it.asset.getAttribute(DEV_EUI).map {DEV_EUI_1.equalsIgnoreCase(it.value.get())}.orElse(false)}- def node2 = assets.find {it.asset.getAttribute(DEV_EUI).map {DEV_EUI_2.equalsIgnoreCase(it.value.get())}.orElse(false)}- assert node1 != null- assert node2 != null+ conditions.eventually {+ assert assets != null+ assert assets.length == 2+ def node1 = assets.find { it.asset.getAttribute(DEV_EUI).map { DEV_EUI_1.equalsIgnoreCase(it.value.get()) }.orElse(false) }+ def node2 = assets.find { it.asset.getAttribute(DEV_EUI).map { DEV_EUI_2.equalsIgnoreCase(it.value.get()) }.orElse(false) }+ assert node1 != null+ assert node2 != null+ def asset1 = assetStorageService.find(new AssetQuery().attributeValue(ATTRIBUTE_NAME_DEV_EUI, DEV_EUI_1.toUpperCase()))+ def asset2 = assetStorageService.find(new AssetQuery().attributeValue(ATTRIBUTE_NAME_DEV_EUI, DEV_EUI_2.toUpperCase()))+ assert asset1 != null+ assert asset2 != null+ }when: "the device sends a LoRaWAN uplink message"- def asset1 = node1.assetdef json = [uplink_message: [f_port: UPLINK_PORT,@@ -239,12 +231,120 @@ class TheThingsStackTest extends Specification implements ManagerContainerTraitthen: "asset attribute values should be updated"conditions.eventually {- def asset = assetStorageService.find(asset1.id)+ def asset = assetStorageService.find(new AssetQuery().attributeValue(ATTRIBUTE_NAME_DEV_EUI, DEV_EUI_1.toUpperCase()))+ assert asset.getAttribute(TEMPERATURE).flatMap {it.value}.map {it == TEMPERATURE_VALUE}.orElse(false)+ assert asset.getAttribute(RELATIVE_HUMIDITY).flatMap {it.value}.map {it == HUMIDITY_VALUE}.orElse(false)+ }++ when: "an asset attribute value is written"+ def asset1 = assetStorageService.find(new AssetQuery().attributeValue(ATTRIBUTE_NAME_DEV_EUI, DEV_EUI_1.toUpperCase()))+ def downlinkMessages = new CopyOnWriteArrayList<String>()+ mqttClient.subscribeWith()+ .topicFilter("v3/${APPLICATION_ID}@${TENANT_ID}/devices/${DEVICE_ID_1}/down/push")+ .callback { Mqtt3Publish publish ->+ downlinkMessages.add(new String(publish.payloadAsBytes))+ }.send()+ assetProcessingService.sendAttributeEvent(new AttributeEvent(asset1.id, SWITCH, true))++ then: "a LoRaWAN downlink message should have been published"+ conditions.eventually {+ assert downlinkMessages.size() == 1+ def downlinkMessage = new JsonSlurper().parseText(downlinkMessages.get(0))+ assert downlinkMessage != null+ assert downlinkMessage.downlinks[0].f_port == DOWNLINK_PORT+ assert downlinkMessage.downlinks[0].frm_payload == "DAE="+ }+ }++ def "TheThingsStack Auto-Discovery Test"() {+ given: "expected conditions"+ def conditions = new PollingConditions(timeout: 10, delay: 0.2)++ when: "the container starts"+ def container = startContainer(defaultConfig(), defaultServices())+ def assetStorageService = container.getService(AssetStorageService.class)+ def assetProcessingService = container.getService(AssetProcessingService.class)+ def agentService = container.getService(AgentService.class)++ and: "a TheThingsStack agent is created"+ def agent = new TheThingsStackAgent("TheThingsStackAgent")+ agent.setRealm(MASTER_REALM)+ agent.setHost(HOST)+ agent.setPort(grpcPort)+ agent.setSecureGRPC(false)+ agent.setMqttHost(HOST)+ agent.setMqttPort(mqttBrokerPort)+ agent.setClientId(CLIENT_ID)+ agent.setApplicationId(APPLICATION_ID)+ agent.setTenantId(TENANT_ID)+ agent.setApiKey(API_KEY)+ agent = assetStorageService.merge(agent)++ then: "the protocol instance for the agent should be created"+ conditions.eventually {+ assert agentService.getProtocolInstance(agent.id) != null+ assert ((TheThingsStackProtocol)agentService.getProtocolInstance(agent.id)) != null+ }++ and: "the gRPC event stream connection should have been established"+ conditions.eventually {+ assert grpcServer.getEventObserverCount() == 1+ }++ when: "the initial gRPC event has been received"+ def streamStartEvent = TheThingsStackGrpcServer.createStreamStartEvent(APPLICATION_ID)+ grpcServer.pushEvent(streamStartEvent)++ then: "the connection status should be CONNECTED"+ conditions.eventually {+ agent = assetStorageService.find(agent.id)+ agent.getAttribute(Agent.STATUS).get().getValue().get() == ConnectionStatus.CONNECTED+ }++ when: "a sensor device sends data for the first time"+ def uplinkEvent = TheThingsStackGrpcServer.createUplinkEvent(APPLICATION_ID, DEVICE_ID_1, DEV_EUI_1, "010A", 1, 101)+ grpcServer.pushEvent(uplinkEvent)++ then: "a new asset should have been auto-discovered"+ conditions.eventually {+ def asset = assetStorageService.find(new AssetQuery().attributeValue(ATTRIBUTE_NAME_DEV_EUI, DEV_EUI_1.toUpperCase()))+ assert asset != null+ }++ when: "a device is created"+ def createEvent = TheThingsStackGrpcServer.createEndDeviceCreateEvent(APPLICATION_ID, DEVICE_ID_2, DEV_EUI_2)+ grpcServer.pushEvent(createEvent)++ then: "a new asset should have been auto-discovered"+ conditions.eventually {+ def asset = assetStorageService.find(new AssetQuery().attributeValue(ATTRIBUTE_NAME_DEV_EUI, DEV_EUI_2.toUpperCase()))+ assert asset != null+ }++ when: "a device sends a LoRaWAN uplink message"+ def json = [+ uplink_message: [+ f_port: UPLINK_PORT,+ decoded_payload: [+ Temperature: TEMPERATURE_VALUE,+ Humidity: HUMIDITY_VALUE+ ]+ ]+ ]+ mqttClient.publishWith()+ .topic("v3/${APPLICATION_ID}@${TENANT_ID}/devices/${DEVICE_ID_1}/up")+ .payload(JsonOutput.toJson(json).bytes)+ .send()++ then: "asset attribute values should be updated"+ conditions.eventually {+ def asset = assetStorageService.find(new AssetQuery().attributeValue(ATTRIBUTE_NAME_DEV_EUI, DEV_EUI_1.toUpperCase()))assert asset.getAttribute(TEMPERATURE).flatMap {it.value}.map {it == TEMPERATURE_VALUE}.orElse(false)assert asset.getAttribute(RELATIVE_HUMIDITY).flatMap {it.value}.map {it == HUMIDITY_VALUE}.orElse(false)}when: "an asset attribute value is written"+ def asset1 = assetStorageService.find(new AssetQuery().attributeValue(ATTRIBUTE_NAME_DEV_EUI, DEV_EUI_1.toUpperCase()))def downlinkMessages = new CopyOnWriteArrayList<String>()mqttClient.subscribeWith().topicFilter("v3/${APPLICATION_ID}@${TENANT_ID}/devices/${DEVICE_ID_1}/down/push")@@ -262,4 +362,257 @@ class TheThingsStackTest extends Specification implements ManagerContainerTraitassert downlinkMessage.downlinks[0].frm_payload == "DAE="}}++ static class TheThingsStackGrpcServer {+ final int port+ final EventsService eventsService+ io.grpc.Server server++ TheThingsStackGrpcServer(int port) {+ this.port = port+ this.eventsService = new EventsService()+ }++ void start() {+ server = ServerBuilder.forPort(port)+ .addService(new DeviceService())+ .addService(eventsService)+ .build()+ .start()+ }++ void stop() {+ if (server != null) {+ server.shutdownNow()+ }+ }++ void pushEvent(Event event) {+ eventsService.pushEvent(event)+ }++ int getEventObserverCount() {+ eventsService.eventObservers.size()+ }++ void shutdownEventStream() {+ eventsService.completeAll()+ }++ private static class DeviceService extends EndDeviceRegistryGrpc.EndDeviceRegistryImplBase {++ @Override+ void list(ListEndDevicesRequest request, StreamObserver<EndDevices> responseObserver) {+ EndDeviceIdentifiers identifiers1 = EndDeviceIdentifiers.newBuilder()+ .setApplicationIds(ApplicationIdentifiers.newBuilder().setApplicationId(APPLICATION_ID))+ .setDeviceId(DEVICE_ID_1)+ .setDevEui(ByteString.copyFrom(DEV_EUI_1.decodeHex()))+ .build()++ EndDevice device1 = EndDevice.newBuilder()+ .setIds(identifiers1)+ .setName("Device 1")+ .putAttributes(THE_THINGS_STACK_ASSET_TYPE_TAG, "TheThingsStackTestAsset")+ .build()++ EndDeviceIdentifiers identifiers2 = EndDeviceIdentifiers.newBuilder()+ .setApplicationIds(ApplicationIdentifiers.newBuilder().setApplicationId(APPLICATION_ID))+ .setDeviceId(DEVICE_ID_2)+ .setDevEui(ByteString.copyFrom(DEV_EUI_2.decodeHex()))+ .build()++ EndDevice device2 = EndDevice.newBuilder()+ .setIds(identifiers2)+ .setName("Device 2")+ .putAttributes(THE_THINGS_STACK_ASSET_TYPE_TAG, "TheThingsStackTestAsset")+ .build()++ EndDevices response = EndDevices.newBuilder()+ .addEndDevices(device1)+ .addEndDevices(device2)+ .build()++ responseObserver.onNext(response)+ responseObserver.onCompleted()+ }++ @Override+ void get(GetEndDeviceRequest request, StreamObserver<EndDevice> responseObserver) {+ def deviceId = request.getEndDeviceIds().getDeviceId()++ if (deviceId == DEVICE_ID_1) {+ EndDeviceIdentifiers identifiers1 = EndDeviceIdentifiers.newBuilder()+ .setApplicationIds(ApplicationIdentifiers.newBuilder().setApplicationId(APPLICATION_ID))+ .setDeviceId(DEVICE_ID_1)+ .setDevEui(ByteString.copyFrom(DEV_EUI_1.decodeHex()))+ .build()++ EndDevice device1 = EndDevice.newBuilder()+ .setIds(identifiers1)+ .setName("Device 1")+ .putAttributes(THE_THINGS_STACK_ASSET_TYPE_TAG, "TheThingsStackTestAsset")+ .build()++ responseObserver.onNext(device1)+ responseObserver.onCompleted()+ } else if (deviceId == DEVICE_ID_2) {+ EndDeviceIdentifiers identifiers2 = EndDeviceIdentifiers.newBuilder()+ .setApplicationIds(ApplicationIdentifiers.newBuilder().setApplicationId(APPLICATION_ID))+ .setDeviceId(DEVICE_ID_2)+ .setDevEui(ByteString.copyFrom(DEV_EUI_2.decodeHex()))+ .build()++ EndDevice device2 = EndDevice.newBuilder()+ .setIds(identifiers2)+ .setName("Device 2")+ .putAttributes(THE_THINGS_STACK_ASSET_TYPE_TAG, "TheThingsStackTestAsset")+ .build()++ responseObserver.onNext(device2)+ responseObserver.onCompleted()+ } else {+ responseObserver.onError(new StatusRuntimeException(Status.NOT_FOUND))+ }+ }+ }++ private static class EventsService extends EventsGrpc.EventsImplBase {+ private final List<StreamObserver<Event>> eventObservers = new CopyOnWriteArrayList<>()++ @Override+ void stream(StreamEventsRequest request, StreamObserver<Event> responseObserver) {+ eventObservers.add(responseObserver)+ }++ void pushEvent(Event event) {+ eventObservers.removeIf { observer ->+ try {+ observer.onNext(event)+ return false+ } catch (Exception e) {+ return true+ }+ }+ }++ void completeAll() {+ eventObservers.each {it.onCompleted()}+ eventObservers.clear()+ }+ }++ static Event createStreamStartEvent(String applicationId) {+ def applicationIds = ApplicationIdentifiers.newBuilder()+ .setApplicationId(applicationId)+ .build()++ def entityIds = EntityIdentifiers.newBuilder()+ .setApplicationIds(applicationIds)+ .build()++ def anyPayload = Any.pack(com.google.protobuf.Empty.getDefaultInstance())++ return Event.newBuilder()+ .setName("events.stream.start")+ .addIdentifiers(entityIds)+ .setData(anyPayload)+ .build()+ }++ static Event createUplinkEvent(String applicationId, String deviceId, String devEui, String payloadData, int fPort, int fCnt) {+ def devEuiBytes = ByteString.copyFrom(devEui.decodeHex())+ def frmPayloadBytes = ByteString.copyFrom(payloadData.decodeHex())+ def uniqueCorrelationId = "mock-corr-id-${System.currentTimeMillis()}-${(int)(Math.random() * 1000)}"++ def applicationIds = ApplicationIdentifiers.newBuilder()+ .setApplicationId(applicationId)+ .build()++ def endDeviceIds = EndDeviceIdentifiers.newBuilder()+ .setApplicationIds(applicationIds)+ .setDeviceId(deviceId)+ .setDevEui(devEuiBytes)+ .build()++ def entityIds = EntityIdentifiers.newBuilder()+ .setDeviceIds(endDeviceIds)+ .build()++ def loraDataRate = Lorawan.LoRaDataRate.newBuilder()+ .setBandwidth(125000)+ .setSpreadingFactor(7)+ .build()++ def dataRate = Lorawan.DataRate.newBuilder()+ .setLora(loraDataRate)+ .build()++ def txSettings = Lorawan.TxSettings.newBuilder()+ .setDataRate(dataRate)+ .setFrequency(868100000)+ .build()++ def rxMetadata = Metadata.RxMetadata.newBuilder()+ .setGatewayIds(GatewayIdentifiers.newBuilder().setGatewayId("mock-gateway-1"))+ .setRssi(-50)+ .setSnr(10)+ .build()++ def applicationUplink = Messages.ApplicationUplink.newBuilder()+ .setFPort(fPort)+ .setFCnt(fCnt)+ .setFrmPayload(frmPayloadBytes)+ .setSettings(txSettings)+ .addRxMetadata(rxMetadata)+ .setReceivedAt(Timestamp.newBuilder().setSeconds((System.currentTimeMillis() / 1000) as long))+ .setConsumedAirtime(Duration.newBuilder().setNanos(50000000))+ .setConfirmed(false)+ .build()++ def applicationUp = Messages.ApplicationUp.newBuilder()+ .setEndDeviceIds(endDeviceIds)+ .setUplinkMessage(applicationUplink)+ .addCorrelationIds(uniqueCorrelationId)+ .build()++ def anyPayload = Any.pack(applicationUp)++ return Event.newBuilder()+ .setName("as.up.data.forward")+ .addCorrelationIds(uniqueCorrelationId)+ .addIdentifiers(entityIds)+ .setData(anyPayload)+ .build()+ }++ static Event createEndDeviceCreateEvent(String applicationId, String deviceId, String devEui) {+ def devEuiBytes = ByteString.copyFrom(devEui.decodeHex())++ def applicationIds = ApplicationIdentifiers.newBuilder()+ .setApplicationId(applicationId)+ .build()++ def endDeviceIds = EndDeviceIdentifiers.newBuilder()+ .setApplicationIds(applicationIds)+ .setDeviceId(deviceId)+ .setDevEui(devEuiBytes)+ .build()++ def entityIds = EntityIdentifiers.newBuilder()+ .setDeviceIds(endDeviceIds)+ .build()++ def endDevice = EndDevice.newBuilder()+ .setIds(endDeviceIds)+ .build()++ def anyPayload = Any.pack(endDevice)++ return Event.newBuilder()+ .setName("end_device.create")+ .addIdentifiers(entityIds)+ .setData(anyPayload)+ .build()+ }+ }}