diff --git a/application/pom.xml b/application/pom.xml index 17b179c767..62d2a49908 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -369,6 +369,10 @@ com.google.firebase firebase-admin + + org.rocksdb + rocksdbjni + diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index 2b19d34e57..d2b3890c70 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -42,6 +42,7 @@ import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; @@ -99,6 +100,8 @@ public abstract class AbstractTbEntityService { protected AssetProfileService assetProfileService; @Autowired protected DeviceProfileService deviceProfileService; + @Autowired + protected EntityService entityService; protected boolean isTestProfile() { return Set.of(this.env.getActiveProfiles()).contains("test"); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 863be23e42..6c2e995b82 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.DeviceId; @@ -118,7 +119,11 @@ public class EntityStateSourcingListener { ApiUsageState apiUsageState = (ApiUsageState) event.getEntity(); tbClusterService.onApiStateChange(apiUsageState, null); } - default -> {} + case CALCULATED_FIELD -> { + onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity()); + } + default -> { + } } } @@ -130,7 +135,7 @@ public class EntityStateSourcingListener { return; } EntityType entityType = entityId.getEntityType(); - if (!tenantId.isSysTenantId() && entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { + if (!tenantId.isSysTenantId() && entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { log.debug("[{}] Ignoring DeleteEntityEvent because tenant does not exist: {}", tenantId, event); return; } @@ -149,7 +154,8 @@ public class EntityStateSourcingListener { case RULE_CHAIN -> { RuleChain ruleChain = (RuleChain) event.getEntity(); if (RuleChainType.CORE.equals(ruleChain.getType())) { - Set referencingRuleChainIds = JacksonUtil.fromString(event.getBody(), new TypeReference<>() {}); + Set referencingRuleChainIds = JacksonUtil.fromString(event.getBody(), new TypeReference<>() { + }); if (referencingRuleChainIds != null) { referencingRuleChainIds.forEach(referencingRuleChainId -> tbClusterService.broadcastEntityStateChangeEvent(tenantId, referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); @@ -177,7 +183,12 @@ public class EntityStateSourcingListener { TbResourceInfo tbResource = (TbResourceInfo) event.getEntity(); tbClusterService.onResourceDeleted(tbResource, null); } - default -> {} + case CALCULATED_FIELD -> { + CalculatedField calculatedField = (CalculatedField) event.getEntity(); + tbClusterService.onCalculatedFieldDeleted(tenantId, calculatedField, null); + } + default -> { + } } } @@ -247,6 +258,15 @@ public class EntityStateSourcingListener { } } + private void onCalculatedFieldUpdate(Object entity, Object oldEntity) { + CalculatedField calculatedField = (CalculatedField) entity; + CalculatedField oldCalculatedField = null; + if (oldEntity instanceof CalculatedField) { + oldCalculatedField = (CalculatedField) oldEntity; + } + tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField); + } + private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { String data = JacksonUtil.toString(JacksonUtil.valueToTree(assignedDevice)); if (data != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java new file mode 100644 index 0000000000..021a9bbc83 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +public class CalculatedFieldCtx { + + private CalculatedFieldId calculatedFieldId; + private EntityId entityId; + private CalculatedFieldState state; + + public CalculatedFieldCtx() { + } + + public CalculatedFieldCtx(CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState state) { + this.calculatedFieldId = calculatedFieldId; + this.entityId = entityId; + this.state = state; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java new file mode 100644 index 0000000000..221c44b94c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +import java.util.Map; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE") +}) +public interface CalculatedFieldState { + + @JsonIgnore + CalculatedFieldType getType(); + + void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index a7c77cf4bc..f2f16de844 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -15,36 +15,64 @@ */ package org.thingsboard.server.service.entitiy.cf; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateEntityId; @@ -52,21 +80,112 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; @TbCoreComponent @Service @Slf4j +@RequiredArgsConstructor public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { - private final Map calculatedFields; - private final Map calculatedFieldLinks; private final CalculatedFieldService calculatedFieldService; + private final AttributesService attributesService; + private final TimeseriesService timeseriesService; + private final RocksDBService rocksDBService; + private ListeningScheduledExecutorService scheduledExecutor; - public DefaultTbCalculatedFieldService(CalculatedFieldService calculatedFieldService) { - this.calculatedFields = calculatedFieldService.findAll().stream().collect(Collectors.toMap(CalculatedField::getId, cf -> cf)); - this.calculatedFieldLinks = calculatedFieldService.findAllCalculatedFieldLinks().stream().collect(Collectors.toMap(CalculatedFieldLink::getId, cfl -> cfl)); - this.calculatedFieldService = calculatedFieldService; + private ListeningExecutorService calculatedFieldExecutor; + private ListeningExecutorService calculatedFieldCallbackExecutor; + + private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); + + @Value("${calculatedField.initFetchPackSize:50000}") + @Getter + private int initFetchPackSize; + + @Value("10") + @Getter + private int defaultCalculatedFieldCheckIntervalInSec; + + @PostConstruct + public void init() { + // from AbstractPartitionBasedService + scheduledExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("calculated-field-scheduled"))); + /// + calculatedFieldExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); + calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); + scheduledExecutor.scheduleWithFixedDelay(this::fetchCalculatedFields, 0, defaultCalculatedFieldCheckIntervalInSec, TimeUnit.SECONDS); } - @Override - public void onMsg() { + @PreDestroy + public void stop() { + // from AbstractPartitionBasedService + if (scheduledExecutor != null) { + scheduledExecutor.shutdown(); + } + /// + if (calculatedFieldExecutor != null) { + calculatedFieldExecutor.shutdownNow(); + } + if (calculatedFieldCallbackExecutor != null) { + calculatedFieldCallbackExecutor.shutdownNow(); + } + } + @Override + public void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); + if (proto.getDeleted()) { + log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); + onCalculatedFieldDelete(calculatedFieldId, callback); + callback.onSuccess(); + } + CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); + if (proto.getUpdated()) { + log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); + boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); + if (!shouldReinit) { + return; + } + } + List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); + if (cf != null) { + EntityId entityId = cf.getEntityId(); + calculatedFields.put(calculatedFieldId, cf); + calculatedFieldLinks.put(calculatedFieldId, links); + switch (entityId.getEntityType()) { + case ASSET, DEVICE -> { + log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); + initializeStateForEntity(tenantId, cf, entityId, callback); + } + case ASSET_PROFILE -> { + log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); + PageDataIterable assetIds = new PageDataIterable<>(pageLink -> + assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); + assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); + } + case DEVICE_PROFILE -> { + log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); + PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> + deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); + deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); + } + default -> + throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); + } + } else { + //Calculated field was probably deleted while message was in queue; + log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); + callback.onSuccess(); + } + callback.onSuccess(); + log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); + } catch (Exception e) { + log.trace("Failed to process calculated field msg: [{}]", proto, e); + callback.onFailure(e); + } } @Override @@ -105,28 +224,70 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } + private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { + try { + calculatedFieldLinks.remove(calculatedFieldId); + calculatedFields.remove(calculatedFieldId); + states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); + List statesToRemove = states.keySet().stream() + .filter(key -> key.startsWith(calculatedFieldId.getId().toString())) + .collect(Collectors.toList()); + rocksDBService.deleteAll(statesToRemove); + } catch (Exception e) { + log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); + callback.onFailure(e); + } + } + + private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { + CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); + boolean shouldReinit = true; + if (hasSignificantChanges(oldCalculatedField, newCalculatedField)) { + onCalculatedFieldDelete(newCalculatedField.getId(), callback); + } else { + calculatedFields.put(newCalculatedField.getId(), newCalculatedField); + callback.onSuccess(); + shouldReinit = false; + } + return shouldReinit; + } + + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + if (oldCalculatedField == null) { + return true; + } + boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); + boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); + CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); + CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); + boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); + boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); + boolean outputExpressionChanged = !oldConfig.getOutput().getExpression().equals(newConfig.getOutput().getExpression()); + + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputExpressionChanged; + } + + private void fetchCalculatedFields() { + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(ctxId, JacksonUtil.fromString(ctx, CalculatedFieldCtx.class))); + states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.startsWith(id.toString()))); + } + private void checkEntityExistence(TenantId tenantId, EntityId entityId) { switch (entityId.getEntityType()) { - case ASSET -> Optional.ofNullable(assetService.findAssetById(tenantId, (AssetId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Asset with id [" + entityId.getId() + "] does not exist.")); - case DEVICE -> Optional.ofNullable(deviceService.findDeviceById(tenantId, (DeviceId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Device with id [" + entityId.getId() + "] does not exist.")); - case ASSET_PROFILE -> - Optional.ofNullable(assetProfileService.findAssetProfileById(tenantId, (AssetProfileId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Asset Profile with id [" + entityId.getId() + "] does not exist.")); - case DEVICE_PROFILE -> - Optional.ofNullable(deviceProfileService.findDeviceProfileById(tenantId, (DeviceProfileId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Device Profile with id [" + entityId.getId() + "] does not exist.")); + case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> + Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) + .orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); default -> throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); } } - private & HasTenantId, I extends EntityId> void checkReferencedEntities(CalculatedFieldConfig calculatedFieldConfig, SecurityUser user) throws ThingsboardException { - List referencedEntityIds = calculatedFieldConfig.getArguments().values().stream() - .map(CalculatedFieldConfig.Argument::getEntityId) - .filter(Objects::nonNull) - .toList(); + private & HasTenantId, I extends EntityId> void checkReferencedEntities(CalculatedFieldConfiguration calculatedFieldConfig, SecurityUser user) throws ThingsboardException { + List referencedEntityIds = calculatedFieldConfig.getReferencedEntities(); for (EntityId referencedEntityId : referencedEntityIds) { validateEntityId(referencedEntityId, id -> "Invalid entity id " + id); E entity = findEntity(user.getTenantId(), referencedEntityId); @@ -137,13 +298,77 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } private & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { - return (E) switch (entityId.getEntityType()) { - case TENANT -> tenantService.findTenantById((TenantId) entityId); - case CUSTOMER -> customerService.findCustomerById(tenantId, (CustomerId) entityId); - case ASSET -> assetService.findAssetById(tenantId, (AssetId) entityId); - case DEVICE -> deviceService.findDeviceById(tenantId, (DeviceId) entityId); - default -> throw new IllegalArgumentException("Calculated fields do not support entity type '" + entityId.getEntityType() + "' for referenced entities."); + return switch (entityId.getEntityType()) { + case TENANT, CUSTOMER, ASSET, DEVICE -> (E) entityService.fetchEntity(tenantId, entityId).orElse(null); + default -> + throw new IllegalArgumentException("Calculated fields do not support entity type '" + entityId.getEntityType() + "' for referenced entities."); + }; + } + + private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { + Map arguments = calculatedField.getConfiguration().getArguments(); + Map argumentValues = new HashMap<>(); + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { + @Override + public void onSuccess(Optional result) { + String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); + argumentValues.put(key, value); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to initialize state for entity: [{}]", entityId, t); + callback.onFailure(t); + } + }, calculatedFieldCallbackExecutor)); + + updateOrInitializeState(calculatedField, entityId, argumentValues); + + } + + private ListenableFuture> fetchArgumentValue(TenantId tenantId, BaseCalculatedFieldConfiguration.Argument argument) { + return switch (argument.getType()) { + case "ATTRIBUTES" -> Futures.transform( + attributesService.find(tenantId, argument.getEntityId(), AttributeScope.SERVER_SCOPE, argument.getKey()), + result -> result.map(entry -> (KvEntry) entry), + MoreExecutors.directExecutor()); + case "TIME_SERIES" -> Futures.transform( + timeseriesService.findLatest(tenantId, argument.getEntityId(), argument.getKey()), + result -> result.map(entry -> (KvEntry) entry), + MoreExecutors.directExecutor()); + default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; } + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { + String ctxId = generateCtxId(calculatedField.getId(), entityId); + CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, + ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); + + CalculatedFieldState state = calculatedFieldCtx.getState(); + if (state != null) { + state.performCalculation(argumentValues, calculatedField.getConfiguration(), false); + } else { + CalculatedFieldState newState = createStateByType(calculatedField.getType()); + newState.performCalculation(argumentValues, calculatedField.getConfiguration(), true); + state = newState; + } + calculatedFieldCtx.setState(state); + + states.put(ctxId, calculatedFieldCtx); + rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.writeValueAsString(calculatedFieldCtx))); + } + + private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { + return switch (calculatedFieldType) { + case SIMPLE -> new SimpleCalculatedFieldState(); + default -> + throw new IllegalArgumentException("Invalid calculated field type '" + calculatedFieldType + "'."); + }; + } + + private String generateCtxId(CalculatedFieldId calculatedFieldId, EntityId entityId) { + return calculatedFieldId.getId() + "_" + entityId.getId(); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java new file mode 100644 index 0000000000..2ba3e9bb4f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java @@ -0,0 +1,95 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.extern.slf4j.Slf4j; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteBatch; +import org.rocksdb.WriteOptions; +import org.springframework.stereotype.Service; +import org.thingsboard.server.utils.RocksDBConfig; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service +@Slf4j +public class RocksDBService { + + private final RocksDB db; + private final WriteOptions writeOptions; + + public RocksDBService(RocksDBConfig config) throws RocksDBException { + this.db = config.getDb(); + this.writeOptions = new WriteOptions().setSync(true); + } + + public void put(String key, String value) { + try { + db.put(writeOptions, key.getBytes(StandardCharsets.UTF_8), value.getBytes(StandardCharsets.UTF_8)); + } catch (RocksDBException e) { + log.error("Failed to store data to RocksDB", e); + } + } + + public void delete(String key) { + try { + db.delete(writeOptions, key.getBytes(StandardCharsets.UTF_8)); + } catch (RocksDBException e) { + log.error("Failed to delete data from RocksDB", e); + } + } + + public void deleteAll(List keys) { + try (WriteBatch batch = new WriteBatch()) { + for (String key : keys) { + batch.delete(key.getBytes(StandardCharsets.UTF_8)); + } + db.write(writeOptions, batch); + } catch (RocksDBException e) { + log.error("Failed to delete data from RocksDB", e); + } + } + + public String get(String key) { + try { + byte[] value = db.get(key.getBytes(StandardCharsets.UTF_8)); + return value != null ? new String(value, StandardCharsets.UTF_8) : null; + } catch (RocksDBException e) { + log.error("Failed to retrieve data from RocksDB", e); + return null; + } + } + + public Map getAll() { + Map map = new HashMap<>(); + try (RocksIterator iterator = db.newIterator()) { + for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { + String key = new String(iterator.key(), StandardCharsets.UTF_8); + String value = new String(iterator.value(), StandardCharsets.UTF_8); + map.put(key, value); + } + } catch (Exception e) { + log.error("Failed to retrieve data from RocksDB", e); + } + return map; + } + +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java new file mode 100644 index 0000000000..8a90893262 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +import java.util.HashMap; +import java.util.Map; + +@Data +public class SimpleCalculatedFieldState implements CalculatedFieldState { + + // TODO: use value object(TsKv) instead of string + Map arguments = new HashMap<>(); + String result; + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SIMPLE; + } + + @Override + public void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation) { + if (initialCalculation) { + // todo: perform initial calculation + this.arguments = argumentValues; + } else { + // todo: perform calculation based on previous data + this.arguments.putAll(argumentValues); + } + this.result = "result"; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java index 2dc1cc35b2..aa77d29702 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -18,11 +18,13 @@ package org.thingsboard.server.service.entitiy.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.security.model.SecurityUser; public interface TbCalculatedFieldService { - void onMsg(); + void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 301f8e8838..f7c8230716 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -38,10 +38,12 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EdgeId; @@ -666,7 +668,8 @@ public class DefaultTbClusterService implements TbClusterService { private void pushDeviceUpdateMessage(TenantId tenantId, EdgeId edgeId, EntityId entityId, EdgeEventActionType action) { log.trace("{} Going to send edge update notification for device actor, device id {}, edge id {}", tenantId, entityId, edgeId); switch (action) { - case ASSIGNED_TO_EDGE -> pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); + case ASSIGNED_TO_EDGE -> + pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); case UNASSIGNED_FROM_EDGE -> { EdgeId relatedEdgeId = findRelatedEdgeIdIfAny(tenantId, entityId); pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), relatedEdgeId), null); @@ -743,4 +746,33 @@ public class DefaultTbClusterService implements TbClusterService { } } + @Override + public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField) { + var created = oldCalculatedField == null; + broadcastEntityChangeToTransport(calculatedField.getTenantId(), calculatedField.getId(), calculatedField, null); + broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + sendCalculatedFieldEvent(calculatedField.getTenantId(), calculatedField.getId(), created, !created, false); + } + + @Override + public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback) { + CalculatedFieldId calculatedFieldId = calculatedField.getId(); + broadcastEntityDeleteToTransport(tenantId, calculatedFieldId, calculatedField.getName(), callback); + sendCalculatedFieldEvent(tenantId, calculatedFieldId, false, false, true); + broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); + } + + private void sendCalculatedFieldEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, boolean added, boolean updated, boolean deleted) { + TransportProtos.CalculatedFieldMsgProto.Builder builder = TransportProtos.CalculatedFieldMsgProto.newBuilder(); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + builder.setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()); + builder.setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()); + builder.setAdded(added); + builder.setUpdated(updated); + builder.setDeleted(deleted); + TransportProtos.CalculatedFieldMsgProto msg = builder.build(); + pushMsgToCore(tenantId, calculatedFieldId, ToCoreMsg.newBuilder().setCalculatedFieldMsg(msg).build(), null); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 2a64dd3388..979c419003 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.NotificationRequestId; import org.thingsboard.server.common.data.id.TenantId; @@ -85,6 +86,7 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -148,6 +150,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig> mainConsumer; private QueueConsumerManager> usageStatsConsumer; @@ -175,7 +178,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = deviceActivityEventsExecutor.submit(() -> calculatedFieldService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); + callback.onFailure(t); + }); + } + private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java b/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java new file mode 100644 index 0000000000..9c3c02f472 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils; + +import jakarta.annotation.PreDestroy; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class RocksDBConfig { + + @Value("${rocksdb.db_path:${java.io.tmpdir}/rocksdb}") + private String dbPath; + private RocksDB db; + + static { + RocksDB.loadLibrary(); + } + + public RocksDB getDb() throws RocksDBException { + if (db == null) { + Options options = new Options().setCreateIfMissing(true); + db = RocksDB.open(options, dbPath); + } + return db; + } + + @PreDestroy + public void close() { + if (db != null) { + db.close(); + db = null; + } + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 106810a7a1..97405f35d3 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -424,6 +424,10 @@ sql: pool_size: "${SQL_RELATIONS_POOL_SIZE:4}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls query_timeout: "${SQL_RELATIONS_QUERY_TIMEOUT_SEC:20}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls +rocksdb: + # Rocksdb path + db_path: "${ROCKS_DB_PATH:${java.io.tmpdir}/rocksdb}" + # Actor system parameters actors: system: diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index affab6ee8b..ebda2cbf82 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -22,7 +22,9 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.security.Authority; @@ -123,7 +125,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { private CalculatedField getCalculatedField(DeviceId deviceId) { CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(deviceId); - calculatedField.setType("Simple"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); calculatedField.setConfiguration(getCalculatedFieldConfig(null)); @@ -131,17 +133,17 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { return calculatedField; } - private CalculatedFieldConfig getCalculatedFieldConfig(EntityId referencedEntityId) { - CalculatedFieldConfig config = new CalculatedFieldConfig(); + private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(referencedEntityId); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java index 60a0ac3bf6..bc1726ff41 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java @@ -38,10 +38,17 @@ import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.entitiy.TbLogEntityActionService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; @@ -81,6 +88,20 @@ public class DefaultTbAlarmServiceTest { protected TbClusterService tbClusterService; @MockBean private EntitiesVersionControlService vcService; + @MockBean + private AccessControlService accessControlService; + @MockBean + private TenantService tenantService; + @MockBean + private AssetService assetService; + @MockBean + private DeviceService deviceService; + @MockBean + private AssetProfileService assetProfileService; + @MockBean + private DeviceProfileService deviceProfileService; + @MockBean + private EntityService entityService; @SpyBean DefaultTbAlarmService service; diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java index b36ca51416..71c295edc9 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java @@ -35,10 +35,17 @@ import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.dao.alarm.AlarmCommentService; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.entitiy.TbLogEntityActionService; import org.thingsboard.server.service.entitiy.alarm.DefaultTbAlarmCommentService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import java.util.UUID; @@ -72,6 +79,20 @@ public class DefaultTbAlarmCommentServiceTest { protected CustomerService customerService; @MockBean protected TbClusterService tbClusterService; + @MockBean + private AccessControlService accessControlService; + @MockBean + private TenantService tenantService; + @MockBean + private AssetService assetService; + @MockBean + private DeviceService deviceService; + @MockBean + private AssetProfileService assetProfileService; + @MockBean + private DeviceProfileService deviceProfileService; + @MockBean + private EntityService entityService; @SpyBean DefaultTbAlarmCommentService service; diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 5112e93da7..f173005107 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.EdgeId; @@ -114,4 +115,8 @@ public interface TbClusterService extends TbQueueClusterService { void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId sourceEdgeId); + void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField); + + void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java index 7252db8097..2a9fe08827 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java @@ -63,6 +63,8 @@ public interface AssetService extends EntityDaoService { PageData findAssetInfosByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink); + PageData findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink); + ListenableFuture> findAssetsByTenantIdAndIdsAsync(TenantId tenantId, List assetIds); void deleteAssetsByTenantId(TenantId tenantId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 7760c49d2d..4bb67f9f0d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -15,12 +15,15 @@ */ package org.thingsboard.server.dao.cf; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.entity.EntityDaoService; import java.util.List; @@ -31,7 +34,11 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId); - List findAll(); + ListenableFuture findCalculatedFieldByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + List findAllCalculatedFields(); + + PageData findAllCalculatedFields(PageLink pageLink); void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); @@ -41,9 +48,15 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedFieldLink findCalculatedFieldLinkById(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); + ListenableFuture findCalculatedFieldLinkByIdAsync(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); + List findAllCalculatedFieldLinks(); - boolean existsByEntityId(TenantId tenantId, EntityId entityId); + List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + PageData findAllCalculatedFieldLinks(PageLink pageLink); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java index 005c740571..0848e5a1cb 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java @@ -75,6 +75,8 @@ public interface DeviceService extends EntityDaoService { PageData findDevicesByTenantIdAndType(TenantId tenantId, String type, PageLink pageLink); + PageData findDeviceIdsByTenantIdAndDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink); + PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType type, PageLink pageLink); long countDevicesByTenantIdAndDeviceProfileIdAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType otaPackageType); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java index 5f522121d7..fc8859eceb 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.entity; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.NameLabelAndCustomerDetails; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -34,6 +35,8 @@ public interface EntityService { Optional fetchEntityCustomerId(TenantId tenantId, EntityId entityId); + Optional> fetchEntity(TenantId tenantId, EntityId entityId); + Optional fetchNameLabelAndCustomerDetails(TenantId tenantId, EntityId entityId); long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..4575e414ac --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java @@ -0,0 +1,161 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import java.util.stream.Collectors; + +@Data +public abstract class BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + @JsonIgnore + private final ObjectMapper mapper = new ObjectMapper(); + + protected Map arguments; + protected Output output; + + public BaseCalculatedFieldConfiguration() { + } + + public BaseCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + BaseCalculatedFieldConfiguration calculatedFieldConfig = toCalculatedFieldConfig(config, entityType, entityId); + this.arguments = calculatedFieldConfig.getArguments(); + this.output = calculatedFieldConfig.getOutput(); + } + + @Override + public List getReferencedEntities() { + return arguments.values().stream() + .map(Argument::getEntityId) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + @Override + public CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId) { + CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); + arguments.values().stream() + .filter(argument -> argument.getEntityId().equals(entityId)) + .forEach(argument -> { + switch (argument.getType()) { + case "ATTRIBUTES": + linkConfiguration.getAttributes().add(argument.getKey()); + break; + case "TIME_SERIES": + linkConfiguration.getTimeSeries().add(argument.getKey()); + break; + } + }); + + return linkConfiguration; + } + + @Override + public JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId) { + ObjectNode configNode = mapper.createObjectNode(); + + ObjectNode argumentsNode = configNode.putObject("arguments"); + arguments.forEach((key, argument) -> { + ObjectNode argumentNode = argumentsNode.putObject(key); + EntityId referencedEntityId = argument.getEntityId(); + if (referencedEntityId != null) { + argumentNode.put("entityType", referencedEntityId.getEntityType().name()); + argumentNode.put("entityId", referencedEntityId.getId().toString()); + } else { + argumentNode.put("entityType", entityType.name()); + argumentNode.put("entityId", entityId.toString()); + } + argumentNode.put("key", argument.getKey()); + argumentNode.put("type", argument.getType()); + argumentNode.put("defaultValue", argument.getDefaultValue()); + }); + + if (output != null) { + ObjectNode outputNode = configNode.putObject("output"); + outputNode.put("type", output.getType()); + outputNode.put("expression", output.getExpression()); + } + + return configNode; + } + + @Data + public static class Argument { + private EntityId entityId; + private String key; + private String type; + private String defaultValue; + } + + @Data + public static class Output { + private String name; + private String type; + private String expression; + } + + private BaseCalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { + if (config == null || !config.isObject()) { + return null; + } + + Map arguments = new HashMap<>(); + JsonNode argumentsNode = config.get("arguments"); + if (argumentsNode != null && argumentsNode.isObject()) { + argumentsNode.fields().forEachRemaining(entry -> { + String key = entry.getKey(); + JsonNode argumentNode = entry.getValue(); + Argument argument = new Argument(); + if (argumentNode.hasNonNull("entityType") && argumentNode.hasNonNull("entityId")) { + String referencedEntityType = argumentNode.get("entityType").asText(); + UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); + argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); + } else { + argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + } + argument.setKey(argumentNode.get("key").asText()); + argument.setType(argumentNode.get("type").asText()); + argument.setDefaultValue(argumentNode.get("defaultValue").asText()); + arguments.put(key, argument); + }); + } + this.setArguments(arguments); + + JsonNode outputNode = config.get("output"); + if (outputNode != null) { + Output output = new Output(); + output.setType(outputNode.get("type").asText()); + output.setExpression(outputNode.get("expression").asText()); + this.setOutput(output); + } + + return this; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index 5c52ad0609..ceb1222fe2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -20,7 +20,11 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; -import org.thingsboard.server.common.data.*; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.HasVersion; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -39,15 +43,15 @@ public class CalculatedField extends BaseData implements HasN @NoXss @Length(fieldName = "type") - private String type; + private CalculatedFieldType type; @NoXss @Length(fieldName = "name") @Schema(description = "User defined name of the calculated field.") private String name; @Schema(description = "Version of calculated field configuration.", example = "0") private int configurationVersion; - @Schema - private transient CalculatedFieldConfig configuration; + @Schema(implementation = SimpleCalculatedFieldConfiguration.class) + private transient CalculatedFieldConfiguration configuration; @Getter @Setter private Long version; @@ -63,7 +67,7 @@ public class CalculatedField extends BaseData implements HasN super(id); } - public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, CalculatedFieldConfig configuration, Long version, CalculatedFieldId externalId) { + public CalculatedField(TenantId tenantId, EntityId entityId, CalculatedFieldType type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version, CalculatedFieldId externalId) { this.tenantId = tenantId; this.entityId = entityId; this.type = type; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java new file mode 100644 index 0000000000..f733c35310 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.JsonNode; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE") +}) +public interface CalculatedFieldConfiguration { + + @JsonIgnore + CalculatedFieldType getType(); + + Map getArguments(); + + BaseCalculatedFieldConfiguration.Output getOutput(); + + @JsonIgnore + List getReferencedEntities(); + + @JsonIgnore + CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId); + + @JsonIgnore + JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java index 2f176b13d2..f9f9ee625f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java @@ -37,7 +37,7 @@ public class CalculatedFieldLink extends BaseData { @Schema(description = "JSON object with the Calculated Field Id. ", accessMode = Schema.AccessMode.READ_ONLY) private CalculatedFieldId calculatedFieldId; @Schema - private transient CalculatedFieldConfig configuration; + private transient CalculatedFieldLinkConfiguration configuration; public CalculatedFieldLink() { super(); @@ -47,7 +47,7 @@ public class CalculatedFieldLink extends BaseData { super(id); } - public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFieldConfig configuration) { + public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFieldLinkConfiguration configuration) { this.tenantId = tenantId; this.entityId = entityId; this.calculatedFieldId = calculatedFieldId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java similarity index 59% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java index b51258b2ec..02d668a67a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java @@ -16,28 +16,14 @@ package org.thingsboard.server.common.data.cf; import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; -import java.util.Map; +import java.util.ArrayList; +import java.util.List; @Data -public class CalculatedFieldConfig { +public class CalculatedFieldLinkConfiguration { - private Map arguments; - private Output output; - - @Data - public static class Argument { - private EntityId entityId; - private String key; - private String type; - private int defaultValue; - } - - @Data - public static class Output { - private String type; - private String expression; - } + private List attributes = new ArrayList<>(); + private List timeSeries = new ArrayList<>(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java new file mode 100644 index 0000000000..89173b35b9 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +public enum CalculatedFieldType { + + SIMPLE, SCRIPT + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..327f9cdc75 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +@Data +public class SimpleCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + public SimpleCalculatedFieldConfiguration() { + super(); + } + + public SimpleCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + super(config, entityType, entityId); + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SIMPLE; + } +} diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 875a531566..95e4fa8601 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1267,6 +1267,16 @@ message ToDeviceActorNotificationMsgProto { DeviceDeleteMsgProto deviceDeleteMsg = 8; } +message CalculatedFieldMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; + bool added = 5; + bool updated = 6; + bool deleted = 7; +} + /** TB Core to Version Control Service */ @@ -1502,6 +1512,7 @@ message ToCoreMsg { DeviceConnectProto deviceConnectMsg = 50; DeviceDisconnectProto deviceDisconnectMsg = 51; DeviceInactivityProto deviceInactivityMsg = 52; + CalculatedFieldMsgProto calculatedFieldMsg = 53; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index 42ab2e2545..8e40ec6d87 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -103,6 +103,16 @@ public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityD */ PageData findAssetInfosByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink); + /** + * Find asset ids by tenantId, assetProfileId and page link. + * + * @param tenantId the tenantId + * @param assetProfileId the assetProfileId + * @param pageLink the page link + * @return the list of asset objects + */ + PageData findAssetIdsByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink); + /** * Find assets by tenantId and assets Ids. * diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index f4c68aa15b..2135792174 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -285,6 +285,15 @@ public class BaseAssetService extends AbstractCachedEntityService findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink) { + log.trace("Executing findAssetIdsByTenantIdAndAssetProfileId, tenantId [{}], assetProfileId [{}]", tenantId, assetProfileId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(assetProfileId, id -> INCORRECT_ASSET_PROFILE_ID + id); + validatePageLink(pageLink); + return assetDao.findAssetIdsByTenantIdAndAssetProfileId(tenantId.getId(), assetProfileId.getId(), pageLink); + } + @Override public ListenableFuture> findAssetsByTenantIdAndIdsAsync(TenantId tenantId, List assetIds) { log.trace("Executing findAssetsByTenantIdAndIdsAsync, tenantId [{}], assetIds [{}]", tenantId, assetIds); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index ecc5aecce1..fd669f7865 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -15,30 +15,38 @@ */ package org.thingsboard.server.dao.cf; +import com.google.common.util.concurrent.ListenableFuture; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; +import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; +import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; -import static org.thingsboard.server.dao.entity.AbstractEntityService.checkConstraintViolation; import static org.thingsboard.server.dao.service.Validator.validateId; +import static org.thingsboard.server.dao.service.Validator.validatePageLink; @Service("CalculatedFieldDaoService") @Slf4j @RequiredArgsConstructor -public class BaseCalculatedFieldService implements CalculatedFieldService { +public class BaseCalculatedFieldService extends AbstractEntityService implements CalculatedFieldService { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; public static final String INCORRECT_CALCULATED_FIELD_ID = "Incorrect calculatedFieldId "; @@ -50,12 +58,14 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { @Override public CalculatedField save(CalculatedField calculatedField) { - calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); + CalculatedField oldCalculatedField = calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); try { TenantId tenantId = calculatedField.getTenantId(); log.trace("Executing save calculated field, [{}]", calculatedField); CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); + eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedCalculatedField.getTenantId()).entityId(savedCalculatedField.getId()) + .entity(savedCalculatedField).oldEntity(oldCalculatedField).created(calculatedField.getId() == null).build()); return savedCalculatedField; } catch (Exception e) { checkConstraintViolation(e, @@ -74,17 +84,49 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { } @Override - public List findAll() { + public ListenableFuture findCalculatedFieldByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findCalculatedFieldByIdAsync [{}]", calculatedFieldId); + validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); + return calculatedFieldDao.findByIdAsync(tenantId, calculatedFieldId.getId()); + } + + @Override + public List findAllCalculatedFields() { log.trace("Executing findAll"); return calculatedFieldDao.findAll(); } + @Override + public PageData findAllCalculatedFields(PageLink pageLink) { + log.trace("Executing findAll, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return calculatedFieldDao.findAll(pageLink); + } + @Override public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); - calculatedFieldDao.removeById(tenantId, calculatedFieldId.getId()); + deleteEntity(tenantId, calculatedFieldId, false); + } + + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + CalculatedField calculatedField = calculatedFieldDao.findById(tenantId, id.getId()); + if (calculatedField == null) { + if (force) { + return; + } else { + throw new IncorrectParameterException("Unable to delete non-existent calculated field."); + } + } + deleteCalculatedField(tenantId, calculatedField); + } + + private void deleteCalculatedField(TenantId tenantId, CalculatedField calculatedField) { + log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedField.getId()); + calculatedFieldDao.removeById(tenantId, calculatedField.getUuidId()); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(calculatedField.getId()).entity(calculatedField).build()); } @Override @@ -99,13 +141,8 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { @Override public CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink) { calculatedFieldLinkDataValidator.validate(calculatedFieldLink, CalculatedFieldLink::getTenantId); - try { - log.trace("Executing save calculated field link, [{}]", calculatedFieldLink); - return calculatedFieldLinkDao.save(tenantId, calculatedFieldLink); - } catch (Exception e) { - checkConstraintViolation(e, "calculated_field_link_unq_key", "Calculated Field for such entity id is already exists!"); - throw e; - } + log.trace("Executing save calculated field link, [{}]", calculatedFieldLink); + return calculatedFieldLinkDao.save(tenantId, calculatedFieldLink); } @Override @@ -116,6 +153,14 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return calculatedFieldLinkDao.findById(tenantId, calculatedFieldLinkId.getId()); } + @Override + public ListenableFuture findCalculatedFieldLinkByIdAsync(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId) { + log.trace("Executing findCalculatedFieldLinkByIdAsync [{}]", calculatedFieldLinkId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldLinkId, id -> "Incorrect calculatedFieldLinkId " + id); + return calculatedFieldLinkDao.findByIdAsync(tenantId, calculatedFieldLinkId.getId()); + } + @Override public List findAllCalculatedFieldLinks() { log.trace("Executing findAllCalculatedFieldLinks"); @@ -123,8 +168,22 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { } @Override - public boolean existsByEntityId(TenantId tenantId, EntityId entityId) { - return calculatedFieldDao.existsByTenantIdAndEntityId(tenantId, entityId); + public List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findAllCalculatedFieldLinksById, calculatedFieldId [{}]", calculatedFieldId); + return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId); + } + + @Override + public ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findAllCalculatedFieldLinksByIdAsync, calculatedFieldId [{}]", calculatedFieldId); + return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldIdAsync(tenantId, calculatedFieldId); + } + + @Override + public PageData findAllCalculatedFieldLinks(PageLink pageLink) { + log.trace("Executing findAllCalculatedFieldLinks, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return calculatedFieldLinkDao.findAll(pageLink); } @Override @@ -132,9 +191,8 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return calculatedFieldDao.findAllByTenantId(tenantId).stream() .filter(calculatedField -> !referencedEntityId.equals(calculatedField.getEntityId())) .map(CalculatedField::getConfiguration) - .map(CalculatedFieldConfig::getArguments) - .flatMap(arguments -> arguments.values().stream()) - .anyMatch(argument -> referencedEntityId.equals(argument.getEntityId())); + .map(CalculatedFieldConfiguration::getReferencedEntities) + .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); } @Override @@ -148,21 +206,22 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { } private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { - CalculatedFieldLink existingLink = (calculatedField.getId() != null) - ? calculatedFieldLinkDao.findCalculatedFieldLinkByCalculatedFieldId(tenantId, calculatedField.getId()) - : null; - - CalculatedFieldLink updatedLink = buildCalculatedFieldLink(tenantId, calculatedField, existingLink); - saveCalculatedFieldLink(tenantId, updatedLink); + List links = buildCalculatedFieldLinks(tenantId, calculatedField); + links.forEach(link -> saveCalculatedFieldLink(tenantId, link)); } - private CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField, CalculatedFieldLink existingLink) { - CalculatedFieldLink link = (existingLink != null) ? existingLink : new CalculatedFieldLink(); - link.setTenantId(tenantId); - link.setEntityId(calculatedField.getEntityId()); - link.setCalculatedFieldId(calculatedField.getId()); - link.setConfiguration(calculatedField.getConfiguration()); - return link; + private List buildCalculatedFieldLinks(TenantId tenantId, CalculatedField calculatedField) { + CalculatedFieldConfiguration cfConfig = calculatedField.getConfiguration(); + return cfConfig.getReferencedEntities().stream() + .map(referencedEntityId -> { + CalculatedFieldLink link = new CalculatedFieldLink(); + link.setTenantId(tenantId); + link.setEntityId(referencedEntityId); + link.setCalculatedFieldId(calculatedField.getId()); + link.setConfiguration(cfConfig.getReferencedEntityConfig(referencedEntityId)); + return link; + }) + .collect(Collectors.toList()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java deleted file mode 100644 index 34dd885e12..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.cf; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; - -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -public class CalculatedFieldConfigUtil { - - public static CalculatedFieldConfig toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { - if (config == null) { - return null; - } - try { - CalculatedFieldConfig calculatedFieldConfig = new CalculatedFieldConfig(); - Map arguments = new HashMap<>(); - - JsonNode argumentsNode = config.get("arguments"); - if (argumentsNode != null && argumentsNode.isObject()) { - argumentsNode.fields().forEachRemaining(entry -> { - String key = entry.getKey(); - JsonNode argumentNode = entry.getValue(); - - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); - if (argumentNode.has("entityType") && argumentNode.has("entityId")) { - String referencedEntityType = argumentNode.get("entityType").asText(); - UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); - argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); - } else { - argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); - } - argument.setKey(argumentNode.get("key").asText()); - argument.setType(argumentNode.get("type").asText()); - - if (argumentNode.has("defaultValue")) { - argument.setDefaultValue(argumentNode.get("defaultValue").asInt()); - } - - arguments.put(key, argument); - }); - } - calculatedFieldConfig.setArguments(arguments); - - JsonNode outputNode = config.get("output"); - if (outputNode != null) { - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); - output.setType(outputNode.get("type").asText()); - output.setExpression(outputNode.get("expression").asText()); - calculatedFieldConfig.setOutput(output); - } - - return calculatedFieldConfig; - - } catch (Exception e) { - throw new IllegalArgumentException("Failed to convert JsonNode to CalculatedFieldConfig", e); - } - } - - public static JsonNode calculatedFieldConfigToJson(CalculatedFieldConfig calculatedFieldConfig, EntityType entityType, UUID entityId) { - if (calculatedFieldConfig == null) { - return null; - } - try { - ObjectNode configNode = JacksonUtil.newObjectNode(); - - ObjectNode argumentsNode = configNode.putObject("arguments"); - calculatedFieldConfig.getArguments().forEach((key, argument) -> { - ObjectNode argumentNode = argumentsNode.putObject(key); - EntityId referencedEntityId = argument.getEntityId(); - if (referencedEntityId != null) { - argumentNode.put("entityType", referencedEntityId.getEntityType().name()); - argumentNode.put("entityId", referencedEntityId.getId().toString()); - } else { - argumentNode.put("entityType", entityType.name()); - argumentNode.put("entityId", entityId.toString()); - } - argumentNode.put("key", argument.getKey()); - argumentNode.put("type", argument.getType()); - argumentNode.put("defaultValue", argument.getDefaultValue()); - }); - - if (calculatedFieldConfig.getOutput() != null) { - ObjectNode outputNode = configNode.putObject("output"); - outputNode.put("type", calculatedFieldConfig.getOutput().getType()); - outputNode.put("expression", calculatedFieldConfig.getOutput().getExpression()); - } - - return configNode; - - } catch (Exception e) { - throw new IllegalArgumentException("Failed to convert CalculatedFieldConfig to JsonNode", e); - } - } - -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index bb52d0e2c2..d1c9fd86bc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -18,18 +18,20 @@ package org.thingsboard.server.dao.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; import java.util.List; public interface CalculatedFieldDao extends Dao { - boolean existsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId); - List findAllByTenantId(TenantId tenantId); List findAll(); + PageData findAll(PageLink pageLink); + List removeAllByEntityId(TenantId tenantId, EntityId entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index c4a06c88a3..34f2129bd7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -15,17 +15,24 @@ */ package org.thingsboard.server.dao.cf; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; import java.util.List; public interface CalculatedFieldLinkDao extends Dao { - CalculatedFieldLink findCalculatedFieldLinkByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + ListenableFuture> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); List findAll(); + PageData findAll(PageLink pageLink); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 2305e4419c..7c82df7703 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -22,7 +22,9 @@ import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; @@ -85,6 +87,16 @@ public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntit */ PageData findDevicesByTenantIdAndType(UUID tenantId, String type, PageLink pageLink); + /** + * Find device ids by tenantId, type and page link. + * + * @param tenantId the tenantId + * @param deviceProfileId the deviceProfileId + * @param pageLink the page link + * @return the list of device objects + */ + PageData findDeviceIdsByTenantIdAndDeviceProfileId(UUID tenantId, UUID deviceProfileId, PageLink pageLink); + PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(UUID tenantId, UUID deviceProfileId, OtaPackageType type, diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index ac6b8330a9..ffebd5f032 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -395,6 +395,15 @@ public class DeviceServiceImpl extends CachedVersionedEntityService findDeviceIdsByTenantIdAndDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink) { + log.trace("Executing findDeviceIdsByTenantIdAndType, tenantId [{}], deviceProfileId [{}], pageLink [{}]", tenantId, deviceProfileId, pageLink); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(deviceProfileId, id -> INCORRECT_DEVICE_PROFILE_ID + id); + validatePageLink(pageLink); + return deviceDao.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId.getId(), deviceProfileId.getId(), pageLink); + } + @Override public PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 3cdcf5e874..5efea9a6df 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -134,6 +134,11 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe return fetchAndConvert(tenantId, entityId, this::getNameLabelAndCustomerDetails); } + @Override + public Optional> fetchEntity(TenantId tenantId, EntityId entityId) { + return fetchAndConvert(tenantId, entityId, Function.identity()); + } + private Optional fetchAndConvert(TenantId tenantId, EntityId entityId, Function, T> converter) { EntityDaoService entityDaoService = entityServiceRegistry.getServiceByEntityType(entityId.getEntityType()); Optional> entityOpt = entityDaoService.findEntity(tenantId, entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 98eb050d28..3c45a81cb5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -24,6 +24,9 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -33,8 +36,6 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.toCalculatedFieldConfig; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION_VERSION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_ID; @@ -56,7 +57,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem private UUID tenantId; @Column(name = CALCULATED_FIELD_ENTITY_TYPE) - private EntityType entityType; + private String entityType; @Column(name = CALCULATED_FIELD_ENTITY_ID) private UUID entityId; @@ -88,12 +89,12 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.setUuid(calculatedField.getUuidId()); this.createdTime = calculatedField.getCreatedTime(); this.tenantId = calculatedField.getTenantId().getId(); - this.entityType = calculatedField.getEntityId().getEntityType(); + this.entityType = calculatedField.getEntityId().getEntityType().name(); this.entityId = calculatedField.getEntityId().getId(); - this.type = calculatedField.getType(); + this.type = calculatedField.getType().name(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); - this.configuration = calculatedFieldConfigToJson(calculatedField.getConfiguration(), entityType, entityId); + this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(EntityType.valueOf(entityType), entityId); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { this.externalId = calculatedField.getExternalId().getId(); @@ -106,10 +107,10 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setCreatedTime(createdTime); calculatedField.setTenantId(TenantId.fromUUID(tenantId)); calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); - calculatedField.setType(type); + calculatedField.setType(CalculatedFieldType.valueOf(type)); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(toCalculatedFieldConfig(configuration, entityType, entityId)); + calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, EntityType.valueOf(entityType), entityId)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); @@ -117,4 +118,13 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem return calculatedField; } + private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + switch (CalculatedFieldType.valueOf(type)) { + case SIMPLE: + return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + default: + throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); + } + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 73a2f9fbdf..b4d6677be4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -22,8 +22,10 @@ import jakarta.persistence.Entity; import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -34,8 +36,6 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.toCalculatedFieldConfig; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_ID; @@ -53,7 +53,7 @@ public class CalculatedFieldLinkEntity extends BaseSqlEntity, Expor @Param("textSearch") String textSearch, Pageable pageable); + @Query("SELECT a.id FROM AssetEntity a " + + "WHERE a.tenantId = :tenantId " + + "AND a.assetProfileId = :assetProfileId " + + "AND (:textSearch IS NULL OR ilike(a.type, CONCAT('%', :textSearch, '%')) = true) ") + Page findAssetIdsByTenantIdAndAssetProfileId(@Param("tenantId") UUID tenantId, + @Param("assetProfileId") UUID assetProfileId, + @Param("textSearch") String textSearch, + Pageable pageable); + @Query("SELECT a FROM AssetEntity a WHERE a.tenantId = :tenantId " + "AND a.customerId = :customerId AND a.type = :type " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 0ef4370f30..db6472f825 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -41,6 +41,7 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; import static org.thingsboard.server.dao.DaoUtil.convertTenantEntityInfosToDto; @@ -159,6 +160,16 @@ public class JpaAssetDao extends JpaAbstractDao implements A DaoUtil.toPageable(pageLink, AssetInfoEntity.assetInfoColumnMap))); } + @Override + public PageData findAssetIdsByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink) { + return DaoUtil.pageToPageData(assetRepository.findAssetIdsByTenantIdAndAssetProfileId( + tenantId, + assetProfileId, + pageLink.getTextSearch(), + DaoUtil.toPageable(pageLink))) + .mapData(AssetId::new); + } + @Override public PageData findAssetsByTenantIdAndCustomerIdAndType(UUID tenantId, UUID customerId, String type, PageLink pageLink) { return DaoUtil.toPageData(assetRepository diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java index fa1c5ce38e..61c4026cca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java @@ -18,10 +18,11 @@ package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; +import java.util.List; import java.util.UUID; public interface CalculatedFieldLinkRepository extends JpaRepository { - CalculatedFieldLinkEntity findByTenantIdAndCalculatedFieldId(UUID tenantId, UUID calculatedFieldId); + List findAllByTenantIdAndCalculatedFieldId(UUID tenantId, UUID calculatedFieldId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java new file mode 100644 index 0000000000..417a468b2c --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -0,0 +1,146 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + +@RequiredArgsConstructor +@Repository +@Slf4j +public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedFieldRepository { + + private final String CF_COUNT_QUERY = "SELECT count(id) FROM calculated_field;"; + private final String CF_QUERY = "SELECT * FROM calculated_field ORDER BY created_time ASC LIMIT %s OFFSET %s"; + + private final String CFL_COUNT_QUERY = "SELECT count(id) FROM calculated_field_link;"; + private final String CFL_QUERY = "SELECT * FROM calculated_field_link ORDER BY created_time ASC LIMIT %s OFFSET %s"; + + private final NamedParameterJdbcTemplate jdbcTemplate; + private final TransactionTemplate transactionTemplate; + + @Override + public PageData findCalculatedFields(Pageable pageable) { + return transactionTemplate.execute(status -> { + long startTs = System.currentTimeMillis(); + int totalElements = jdbcTemplate.queryForObject(CF_COUNT_QUERY, Collections.emptyMap(), Integer.class); + log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); + startTs = System.currentTimeMillis(); + List> rows = jdbcTemplate.queryForList(String.format(CF_QUERY, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); + log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); + int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; + boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); + var data = rows.stream().map(row -> { + + UUID id = (UUID) row.get("id"); + long createdTime = (long) row.get("created_time"); + UUID tenantId = (UUID) row.get("tenant_id"); + EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); + UUID entityId = (UUID) row.get("entity_id"); + CalculatedFieldType type = CalculatedFieldType.valueOf((String) row.get("type")); + String name = (String) row.get("name"); + int configurationVersion = (int) row.get("configuration_version"); + JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); + long version = (long) row.get("version"); + Object externalIdObj = row.get("external_id"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setId(new CalculatedFieldId(id)); + calculatedField.setCreatedTime(createdTime); + calculatedField.setTenantId(new TenantId(tenantId)); + calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedField.setType(type); + calculatedField.setName(name); + calculatedField.setConfigurationVersion(configurationVersion); + calculatedField.setConfiguration(readCalculatedFieldConfiguration(type, configuration, entityType, entityId)); + calculatedField.setVersion(version); + calculatedField.setExternalId(externalIdObj != null ? new CalculatedFieldId(UUID.fromString((String) externalIdObj)) : null); + + return calculatedField; + }).collect(Collectors.toList()); + return new PageData<>(data, totalPages, totalElements, hasNext); + }); + } + + @Override + public PageData findCalculatedFieldLinks(Pageable pageable) { + return transactionTemplate.execute(status -> { + long startTs = System.currentTimeMillis(); + int totalElements = jdbcTemplate.queryForObject(CFL_COUNT_QUERY, Collections.emptyMap(), Integer.class); + log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); + startTs = System.currentTimeMillis(); + List> rows = jdbcTemplate.queryForList(String.format(CFL_QUERY, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); + log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); + int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; + boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); + var data = rows.stream().map(row -> { + + UUID id = (UUID) row.get("id"); + long createdTime = (long) row.get("created_time"); + UUID tenantId = (UUID) row.get("tenant_id"); + EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); + UUID entityId = (UUID) row.get("entity_id"); + UUID calculatedFieldId = (UUID) row.get("calculated_field_id"); + JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); + + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); + calculatedFieldLink.setId(new CalculatedFieldLinkId(id)); + calculatedFieldLink.setCreatedTime(createdTime); + calculatedFieldLink.setTenantId(new TenantId(tenantId)); + calculatedFieldLink.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedFieldLink.setCalculatedFieldId(new CalculatedFieldId(calculatedFieldId)); + calculatedFieldLink.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldLinkConfiguration.class)); + + return calculatedFieldLink; + }).collect(Collectors.toList()); + return new PageData<>(data, totalPages, totalElements, hasNext); + }); + } + + private CalculatedFieldConfiguration readCalculatedFieldConfiguration(CalculatedFieldType type, JsonNode config, EntityType entityType, UUID entityId) { + switch (type) { + case SIMPLE: + return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + default: + throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); + } + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 4bf4c1ead7..bdc701070d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -24,6 +24,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; @@ -40,11 +42,7 @@ import java.util.UUID; public class JpaCalculatedFieldDao extends JpaAbstractDao implements CalculatedFieldDao { private final CalculatedFieldRepository calculatedFieldRepository; - - @Override - public boolean existsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId) { - return calculatedFieldRepository.existsByTenantIdAndEntityId(tenantId.getId(), entityId.getId()); - } + private final NativeCalculatedFieldRepository nativeCalculatedFieldRepository; @Override public List findAllByTenantId(TenantId tenantId) { @@ -56,6 +54,12 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAll(PageLink pageLink) { + log.debug("Try to find calculated fields by pageLink [{}]", pageLink); + return nativeCalculatedFieldRepository.findCalculatedFields(DaoUtil.toPageable(pageLink)); + } + @Override @Transactional public List removeAllByEntityId(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index f584a8d76c..417b529dc9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.sql.cf; +import com.google.common.util.concurrent.ListenableFuture; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; @@ -23,6 +24,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; @@ -39,10 +42,16 @@ import java.util.UUID; public class JpaCalculatedFieldLinkDao extends JpaAbstractDao implements CalculatedFieldLinkDao { private final CalculatedFieldLinkRepository calculatedFieldLinkRepository; + private final NativeCalculatedFieldRepository nativeCalculatedFieldRepository; @Override - public CalculatedFieldLink findCalculatedFieldLinkByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - return DaoUtil.getData(calculatedFieldLinkRepository.findByTenantIdAndCalculatedFieldId(tenantId.getId(), calculatedFieldId.getId())); + public List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAllByTenantIdAndCalculatedFieldId(tenantId.getId(), calculatedFieldId.getId())); + } + + @Override + public ListenableFuture> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + return service.submit(() -> findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId)); } @Override @@ -50,6 +59,12 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao findAll(PageLink pageLink) { + log.debug("Try to find calculated field links by pageLink [{}]", pageLink); + return nativeCalculatedFieldRepository.findCalculatedFieldLinks(DaoUtil.toPageable(pageLink)); + } + @Override protected Class getEntityClass() { return CalculatedFieldLinkEntity.class; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java new file mode 100644 index 0000000000..76fed8c311 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import org.springframework.data.domain.Pageable; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.page.PageData; + +public interface NativeCalculatedFieldRepository { + + PageData findCalculatedFields(Pageable pageable); + + PageData findCalculatedFieldLinks(Pageable pageable); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java index c55210b606..5093da78c5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java @@ -81,6 +81,14 @@ public interface DeviceRepository extends JpaRepository, Exp @Param("textSearch") String textSearch, Pageable pageable); + @Query("SELECT d.id FROM DeviceEntity d WHERE d.tenantId = :tenantId " + + "AND d.deviceProfileId = :deviceProfileId " + + "AND (:textSearch IS NULL OR ilike(d.type, CONCAT('%', :textSearch, '%')) = true)") + Page findIdsByTenantIdAndDeviceProfileId(@Param("tenantId") UUID tenantId, + @Param("deviceProfileId") UUID deviceProfileId, + @Param("textSearch") String textSearch, + Pageable pageable); + @Query("SELECT d FROM DeviceEntity d WHERE d.tenantId = :tenantId " + "AND d.deviceProfileId = :deviceProfileId " + "AND d.firmwareId IS NULL") diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 48bb998016..bf0def9fca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -173,6 +173,17 @@ public class JpaDeviceDao extends JpaAbstractDao implement DaoUtil.toPageable(pageLink))); } + @Override + public PageData findDeviceIdsByTenantIdAndDeviceProfileId(UUID tenantId, UUID deviceProfileId, PageLink pageLink) { + return DaoUtil.pageToPageData( + deviceRepository.findIdsByTenantIdAndDeviceProfileId( + tenantId, + deviceProfileId, + pageLink.getTextSearch(), + DaoUtil.toPageable(pageLink))) + .mapData(DeviceId::new); + } + @Override public PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(UUID tenantId, UUID deviceProfileId, diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 790317fcb0..3c1e124417 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -925,9 +925,7 @@ CREATE TABLE IF NOT EXISTS calculated_field_link ( tenant_id uuid NOT NULL, entity_type VARCHAR(32), entity_id uuid NOT NULL, --- target_id uuid NOT NULL, calculated_field_id uuid NOT NULL, - configuration varchar(1000000), - CONSTRAINT calculated_field_link_unq_key UNIQUE (entity_id, calculated_field_id), + configuration varchar(10000), CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE ); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index ba0c73fb09..43b51ce2ac 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -31,7 +31,8 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -874,19 +875,19 @@ public class AssetServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("Simple"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setEntityId(savedAssetWithCf.getId()); - CalculatedFieldConfig config = new CalculatedFieldConfig(); + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(savedAsset.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index c4ad2856a9..d5d025a46d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -24,10 +24,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; @@ -128,16 +128,6 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { assertThat(calculatedFieldService.findById(tenantId, savedCalculatedField.getId())).isNull(); } - @Test - public void testSaveCalculatedFieldLinkIfCalculatedFieldForSuchEntityExists() { - CalculatedField savedCalculatedField = saveValidCalculatedField(); - CalculatedFieldLink calculatedFieldLink = getCalculatedFieldLink(savedCalculatedField); - - assertThatThrownBy(() -> calculatedFieldService.saveCalculatedFieldLink(tenantId, calculatedFieldLink)) - .isInstanceOf(DataValidationException.class) - .hasMessage("Calculated Field for such entity id is already exists!"); - } - private CalculatedField saveValidCalculatedField() { Device device = createTestDevice(); CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); @@ -148,7 +138,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setEntityId(entityId); - calculatedField.setType("Simple"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); @@ -156,26 +146,17 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { return calculatedField; } - private CalculatedFieldLink getCalculatedFieldLink(CalculatedField calculatedField) { - CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); - calculatedFieldLink.setTenantId(tenantId); - calculatedFieldLink.setEntityId(calculatedField.getEntityId()); -// calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); - calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); - return calculatedFieldLink; - } - - private CalculatedFieldConfig getCalculatedFieldConfig(EntityId referencedEntityId) { - CalculatedFieldConfig config = new CalculatedFieldConfig(); + private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(referencedEntityId); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 548a5bfcf5..1c5e0d8f49 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -32,7 +32,8 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -369,19 +370,19 @@ public class CustomerServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("Simple"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setEntityId(savedAsset.getId()); - CalculatedFieldConfig config = new CalculatedFieldConfig(); + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(savedCustomer.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 16bb08350b..8f1f16e631 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -40,7 +40,8 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -1212,19 +1213,19 @@ public class DeviceServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("Simple"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setEntityId(deviceWithCf.getId()); - CalculatedFieldConfig config = new CalculatedFieldConfig(); + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(device.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); @@ -1241,5 +1242,4 @@ public class DeviceServiceTest extends AbstractServiceTest { calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); } - } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java index 6d0eb0ad38..10df39e04b 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java @@ -20,6 +20,7 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.SpyBean; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.cf.CalculatedFieldDao; @@ -44,7 +45,7 @@ public class CalculatedFieldDataValidatorTest { @Test public void testUpdateNonExistingCalculatedField() { CalculatedField calculatedField = new CalculatedField(CALCULATED_FIELD_ID); - calculatedField.setType("Simple"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setName("Test"); given(calculatedFieldDao.findById(TENANT_ID, CALCULATED_FIELD_ID.getId())).willReturn(null); diff --git a/pom.xml b/pom.xml index a1d5f26ecf..9b835516fc 100755 --- a/pom.xml +++ b/pom.xml @@ -166,6 +166,8 @@ 1.6.1 2.19.0 9.2.0 + + 9.4.0 @@ -2272,6 +2274,11 @@ metadata-extractor ${drewnoakes-metadata-extractor.version} + + org.rocksdb + rocksdbjni + ${rocksdbjni.version} +