100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Replace value descriptor schema cache key mechanism with ETags (#2547)
Koen3b5b00acfd78
diff --git a/manager/src/main/java/org/openremote/manager/asset/AssetModelResourceImpl.java b/manager/src/main/java/org/openremote/manager/asset/AssetModelResourceImpl.javaindex 0094d486d..4ce0ffb48 100644--- a/manager/src/main/java/org/openremote/manager/asset/AssetModelResourceImpl.java+++ b/manager/src/main/java/org/openremote/manager/asset/AssetModelResourceImpl.java@@ -20,18 +20,21 @@package org.openremote.manager.asset;import jakarta.ws.rs.WebApplicationException;+import jakarta.ws.rs.core.EntityTag;+import jakarta.ws.rs.core.Request;+import jakarta.ws.rs.core.Response;+import org.openremote.container.timer.TimerService;import org.openremote.manager.security.ManagerIdentityService;import org.openremote.manager.web.ManagerWebResource;import org.openremote.model.asset.AssetDescriptor;import org.openremote.model.asset.AssetModelResource;import org.openremote.model.http.RequestParams;+import org.openremote.model.util.ValueUtil.SchemaResult;import org.openremote.model.asset.AssetTypeInfo;import org.openremote.model.value.MetaItemDescriptor;import org.openremote.model.value.ValueDescriptor;-import com.fasterxml.jackson.databind.JsonNode;-import java.util.Map;import static jakarta.ws.rs.core.Response.Status.*;@@ -71,11 +74,21 @@ public class AssetModelResourceImpl extends ManagerWebResource implements AssetM}@Override- public JsonNode getValueDescriptorSchema(RequestParams requestParams, String name, String hash) {- JsonNode schema = assetModelService.getValueDescriptorSchema(name);- if (schema == null) {+ public Response getValueDescriptorSchema(String name, Request request) {+ SchemaResult result = assetModelService.getValueDescriptorSchema(name);++ if (result == null) {throw new WebApplicationException(NOT_FOUND);}- return schema;++ EntityTag etag = new EntityTag(result.hash());++ Response.ResponseBuilder builder = request.evaluatePreconditions(etag);+ if (builder != null) {+ // If the ETag matches, return 304 (Not Modified)+ return builder.build();+ }++ return Response.ok(result.schema()).tag(etag).build();}}diff --git a/manager/src/main/java/org/openremote/manager/asset/AssetModelService.java b/manager/src/main/java/org/openremote/manager/asset/AssetModelService.javaindex 6d967cd8d..08b24b38d 100644--- a/manager/src/main/java/org/openremote/manager/asset/AssetModelService.java+++ b/manager/src/main/java/org/openremote/manager/asset/AssetModelService.java@@ -46,14 +46,13 @@ import org.openremote.model.attribute.MetaMap;import org.openremote.model.syslog.SyslogCategory;import org.openremote.model.util.TextUtil;import org.openremote.model.util.ValueUtil;+import org.openremote.model.util.ValueUtil.SchemaResult;import org.openremote.model.value.AttributeDescriptor;import org.openremote.model.value.MetaItemDescriptor;import org.openremote.model.value.ValueDescriptor;import java.io.IOException;-import com.fasterxml.jackson.databind.JsonNode;-import java.nio.file.Files;import java.nio.file.Path;import java.util.*;@@ -376,7 +375,7 @@ public class AssetModelService extends RouteBuilder implements ContainerService,return ValueUtil.getMetaItemDescriptors();}- public JsonNode getValueDescriptorSchema(String name) {+ public SchemaResult getValueDescriptorSchema(String name) {return ValueUtil.getValueDescriptorSchema(name);}diff --git a/manager/src/main/java/org/openremote/manager/system/StatusResourceImpl.java b/manager/src/main/java/org/openremote/manager/system/StatusResourceImpl.javaindex a2ae60326..c7e1fac64 100644--- a/manager/src/main/java/org/openremote/manager/system/StatusResourceImpl.java+++ b/manager/src/main/java/org/openremote/manager/system/StatusResourceImpl.java@@ -24,7 +24,6 @@ import org.openremote.manager.security.ManagerIdentityService;import org.openremote.model.Container;import org.openremote.model.system.HealthStatusProvider;import org.openremote.model.system.StatusResource;-import org.openremote.model.util.ValueUtil;import java.util.HashMap;import java.util.List;@@ -49,8 +48,7 @@ public class StatusResourceImpl implements StatusResource {serverInfo = Map.of("version", version,- "authServerUrl", authServerUrl,- "valueDescriptorSchemaHashes", ValueUtil.getValueDescriptorSchemaHashes()+ "authServerUrl", authServerUrl);LOG.info("Starting OpenRemote version: v"+version);diff --git a/model/src/main/java/org/openremote/model/asset/AssetModelResource.java b/model/src/main/java/org/openremote/model/asset/AssetModelResource.javaindex 23be3c965..79adad397 100644--- a/model/src/main/java/org/openremote/model/asset/AssetModelResource.java+++ b/model/src/main/java/org/openremote/model/asset/AssetModelResource.java@@ -19,16 +19,18 @@*/package org.openremote.model.asset;-import com.fasterxml.jackson.databind.JsonNode;import io.swagger.v3.oas.annotations.Operation;+import io.swagger.v3.oas.annotations.Parameter;import io.swagger.v3.oas.annotations.tags.Tag;+import jakarta.ws.rs.core.Context;+import jakarta.ws.rs.core.Request;+import org.jboss.resteasy.annotations.cache.Cache;import org.openremote.model.http.RequestParams;-import org.openremote.model.system.StatusResource;import org.openremote.model.value.MetaItemDescriptor;import org.openremote.model.value.ValueDescriptor;-import org.jboss.resteasy.annotations.cache.Cache;import jakarta.ws.rs.*;+import jakarta.ws.rs.core.Response;import java.util.Map;@@ -105,13 +107,13 @@ public interface AssetModelResource {/*** Retrieve the JSON Schema for a {@link ValueDescriptor} available in this system. A value descriptor schema is only meant to be retrieved- * once per client. Either when a new {@code hash} or {@code name} are requested. The HTTP client should cache the response based on the- * {@code Cache-Control} header. The {@code hash} for actively cached descriptors can be found by requesting the {@link StatusResource#getInfo()} endpoint.+ * once per client. Either when a new {@code name} is requested or the "If-None-Match" header does not match the current ETag. The HTTP client should+ * use the provided ETag to cache the response.*/@GET@Path("getValueDescriptorSchema")@Produces(APPLICATION_JSON)- @Cache(maxAge = 31536000) // A 1-year cache, the supplied hash parameter serves as cache key on the client- @Operation(operationId = "getValueDescriptorSchema", summary = "Retrieve the valueDescriptor JSON Schema.")- JsonNode getValueDescriptorSchema(@BeanParam RequestParams requestParams, @QueryParam("name") String name, @QueryParam("hash") String hash);+ @Cache(noCache = true)+ @Operation(operationId = "getValueDescriptorSchema", summary = "Retrieve the JSON Schema of the specified value descriptor")+ Response getValueDescriptorSchema(@QueryParam("name") String name, @Context Request request);}diff --git a/model/src/main/java/org/openremote/model/util/ValueUtil.java b/model/src/main/java/org/openremote/model/util/ValueUtil.javaindex 60b0b8bb6..021360ad0 100644--- a/model/src/main/java/org/openremote/model/util/ValueUtil.java+++ b/model/src/main/java/org/openremote/model/util/ValueUtil.java@@ -27,7 +27,6 @@ import com.fasterxml.jackson.databind.deser.BeanDeserializerModifier;import com.fasterxml.jackson.databind.jsontype.NamedType;import com.fasterxml.jackson.databind.module.SimpleModule;import com.fasterxml.jackson.databind.node.NullNode;-import com.fasterxml.jackson.databind.node.ObjectNode;import com.fasterxml.jackson.databind.node.TextNode;import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;@@ -153,6 +152,17 @@ public class ValueUtil {}}+ // TODO: consider canonicalization based on: https://www.rfc-editor.org/rfc/rfc8785+ public record SchemaResult(String schema, String hash) {+ public SchemaResult (String schema) {+ this(schema, hash(schema));+ }+ private static String hash(String schema) {+ MessageDigest md = MESSAGE_DIGEST_HOLDER.get(); md.reset();+ return bytesToHexString(md.digest(schema.getBytes(StandardCharsets.UTF_8)));+ }+ }+// Preload the Standard model provider so it takes priority over otherspublic static Logger LOG = SyslogCategory.getLogger(MODEL_AND_VALUES, ValueUtil.class);public static ObjectMapper JSON = configureObjectMapper(new ObjectMapper());@@ -162,20 +172,17 @@ public class ValueUtil {protected static Map<String, Class<? extends AgentLink<?>>> agentTypeMap = new HashMap<>();protected static Map<String, MetaItemDescriptor<?>> metaItemDescriptors = new HashMap<>();protected static Map<String, ValueDescriptor<?>> valueDescriptors = new HashMap<>();- protected static Map<String, ObjectNode> valueDescriptorSchemas = new ConcurrentHashMap<>();- protected static Map<String, String> valueDescriptorSchemaHashes = new ConcurrentHashMap<>();+ protected static Map<String, SchemaResult> valueDescriptorSchemas = new ConcurrentHashMap<>();protected static Validator validator;protected static SchemaGenerator generator;- protected static MessageDigest md;-- static {+ private static final ThreadLocal<MessageDigest> MESSAGE_DIGEST_HOLDER = ThreadLocal.withInitial(() -> {try {- md = MessageDigest.getInstance("MD5");+ return MessageDigest.getInstance("MD5");} catch (NoSuchAlgorithmException e) {LOG.severe("MD5 algorithm not supported");throw new RuntimeException(e);}- }+ });public static ObjectMapper configureObjectMapper(ObjectMapper objectMapper) {objectMapper@@ -656,10 +663,6 @@ public class ValueUtil {return Optional.ofNullable(metaItemDescriptors.get(name));}- public static Map<String, String> getValueDescriptorSchemaHashes() {- return valueDescriptorSchemaHashes;- }-public static Map<String, ValueDescriptor<?>> getValueDescriptors() {return valueDescriptors;}@@ -953,23 +956,16 @@ public class ValueUtil {valueDescriptorSchemas.putAll(metaItems.values().stream().map(AbstractNameValueDescriptorHolder::getType).filter(v -> v.getJsonType().equals("object") || v.getArrayDimensions() != null && v.getArrayDimensions() > 0)- .collect(Collectors.toMap(ValueDescriptor::getName, vd -> (ObjectNode) getSchema(vd.getType()))));- valueDescriptorSchemaHashes.putAll(valueDescriptorSchemas.entrySet().stream().collect(Collectors.toMap(- Map.Entry::getKey,- v -> hash(v.getValue().toString())- )));+ .collect(Collectors.toMap(ValueDescriptor::getName, vd -> new SchemaResult(getSchema(vd.getType()).toString())))+ );}- public static JsonNode getValueDescriptorSchema(String name) {+ public static SchemaResult getValueDescriptorSchema(String name) {if (ValueUtil.getValueDescriptor(name).isEmpty()) {return null;}ValueDescriptor<?> vd = ValueUtil.getValueDescriptor(name).get();- return valueDescriptorSchemas.computeIfAbsent(vd.getName(), key -> {- ObjectNode schema = (ObjectNode) ValueUtil.getSchema(vd.getType());- valueDescriptorSchemaHashes.put(vd.getName(), hash(schema.toString()));- return schema;- });+ return valueDescriptorSchemas.computeIfAbsent(vd.getName(), key -> new SchemaResult(getSchema(vd.getType()).toString()));}protected static Class<?>[] getAgentLinkClasses() {@@ -982,10 +978,6 @@ public class ValueUtil {.toArray(Class<?>[]::new);}- public static String hash(String v) {- return bytesToHexString(md.digest(v.getBytes(StandardCharsets.UTF_8)));- }-/*** Validates the supplied object using standard JSR-380 bean validation; therefore any type passed in here must* follow the JSR-380 annotation requirements.diff --git a/test/src/test/groovy/org/openremote/test/gateway/GatewayTest.groovy b/test/src/test/groovy/org/openremote/test/gateway/GatewayTest.groovyindex 2d02f2a4d..67dc082bb 100644--- a/test/src/test/groovy/org/openremote/test/gateway/GatewayTest.groovy+++ b/test/src/test/groovy/org/openremote/test/gateway/GatewayTest.groovy@@ -1438,7 +1438,6 @@ class GatewayTest extends Specification implements ManagerContainerTrait {assert info != nullassert info.size() > 0- assert info.valueDescriptorSchemaHashes == ValueUtil.getValueDescriptorSchemaHashes()}diff --git a/test/src/test/groovy/org/openremote/test/model/AssetModelTest.groovy b/test/src/test/groovy/org/openremote/test/model/AssetModelTest.groovyindex 0da15a19e..d3e15b2cd 100644--- a/test/src/test/groovy/org/openremote/test/model/AssetModelTest.groovy+++ b/test/src/test/groovy/org/openremote/test/model/AssetModelTest.groovy@@ -610,27 +610,32 @@ class AssetModelTest extends Specification implements ManagerContainerTrait {}@Unroll- def "Get valueDescriptor schema for #name"(String name, Class<?> clazz) {+ def "Get valueDescriptor schema and hash for #name"(String name, Class<?> clazz, String hash) {given: "required services are setup"def assetModelService = container.getService(AssetModelService.class)when: "we ask to generate a schema for #clazz"- def schema = assetModelService.getValueDescriptorSchema(name)- def expected = ValueUtil.getSchema(clazz)+ def result = assetModelService.getValueDescriptorSchema(name)+ def expected = ValueUtil.getSchema(clazz).toString()then: "the schema to be the same"- Objects.equals(schema, expected)+ Objects.equals(result.schema(), expected)++ and: "the hash to be the same"+ if (!hash.empty) {+ Objects.equals(result.hash(), hash)+ }where:- name | clazz- "text" | String- "text[]" | String[]- "text[][]" | String[][]- "agentLink" | AgentLink- "attributeLink" | AttributeLink- "valueConstraint[]" | ValueConstraint[]- "forecastConfiguration" | ForecastConfiguration- "valueFormat" | ValueFormat+ name | clazz | hash+ "text" | String | "04818E8A57A9EBA38CA5457DA1907FEF"+ "text[]" | String[] | "A843B49121780A9A4DCA0FB5604CDA1B"+ "text[][]" | String[][] | "2FD46740DD360941363BDA6D7F2D9EF2"+ "agentLink" | AgentLink | "" // These are likely to change+ "attributeLink" | AttributeLink | "" // so we skip them.+ "valueConstraint[]" | ValueConstraint[] | ""+ "forecastConfiguration" | ForecastConfiguration | ""+ "valueFormat" | ValueFormat | ""}def "Get unknown valueDescriptor schema"() {diff --git a/ui/component/or-attribute-input/src/index.ts b/ui/component/or-attribute-input/src/index.tsindex a56a03f41..f9725ad02 100644--- a/ui/component/or-attribute-input/src/index.ts+++ b/ui/component/or-attribute-input/src/index.ts@@ -3,7 +3,7 @@ import {customElement, property} from "lit/decorators.js";import {ifDefined} from "lit/directives/if-defined.js";import {until} from "lit/directives/until.js";import {createRef, Ref, ref} from 'lit/directives/ref.js';-import {i18next, translate} from "@openremote/or-translate";+import { i18next, translate } from "@openremote/or-translate";import {Attribute,AttributeDescriptor,@@ -116,8 +116,35 @@ export function getHelperText(sending: boolean, error: boolean, timestamp: numbeconst jsonFormsAttributeRenderers = [...StandardRenderers, agentIdRendererRegistryEntry, schedulerRendererRegistryEntry];-let valueDescriptorSchemaHashes: Record<string, string>-const schemas = new Map<string, any>()+const schemas = new Map<string, unknown>();+const inflightRequests = new Map<string, Promise<unknown>>();++async function getSchema(valueDescriptor: ValueDescriptor) {+ const descriptor = valueDescriptor.name + "[]".repeat(valueDescriptor.arrayDimensions ?? 0);++ if (schemas.has(descriptor)) {+ return schemas.get(descriptor);+ }++ // Coalesce inflight requests+ // Firefox won't guarantee subsequent requests provide the+ // "If-None-Match" request header.+ if (inflightRequests.has(descriptor)) {+ return inflightRequests.get(descriptor);+ }++ const promise = manager.rest.api.AssetModelResource+ .getValueDescriptorSchema({ name: descriptor })+ .then(response => {+ const schema = response.data;+ schemas.set(descriptor, schema);+ return schema;+ })+ .finally(() => inflightRequests.delete(descriptor));++ inflightRequests.set(descriptor, promise);+ return promise;+}export const jsonFormsInputTemplateProvider: (fallback: ValueInputProvider) => ValueInputProviderGenerator = (fallback) => (assetDescriptor, valueHolder, valueHolderDescriptor, valueDescriptor, valueChangeNotifier, options) => {if (Util.isComplexMetaItem(valueDescriptor)) {@@ -153,25 +180,7 @@ export const jsonFormsInputTemplateProvider: (fallback: ValueInputProvider) => V}initialised = true;- if (!valueDescriptorSchemaHashes) {- const response = await manager.rest.api.StatusResource.getInfo();- valueDescriptorSchemaHashes = response.data.valueDescriptorSchemaHashes;- }-- const descriptor = valueDescriptor.name + "[]".repeat(valueDescriptor.arrayDimensions ?? 0);- const hash = valueDescriptorSchemaHashes[descriptor];-- if (!schema && !schemas.has(descriptor)) {- const response = await manager.rest.api.AssetModelResource.getValueDescriptorSchema({- name: descriptor,- hash,- }, { headers: !hash ? { "Cache-Control": "no-cache" } : {} });- schema = response.data;- schemas.set(descriptor, schema);- // label ||= schema.title- } else {- schema = schemas.get(descriptor);- }+ schema = await getSchema(valueDescriptor);if (jsonForms.value && loadingWrapper.value) {const forms = jsonForms.value;