From 86dd8e725bf701f0c0fe561c3e2b6e8729235305 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 11 Nov 2024 17:29:24 +0200 Subject: [PATCH 01/13] added onCalculatedFieldAdded implementation --- .../entitiy/AbstractTbEntityService.java | 3 + .../entitiy/cf/CalculatedFieldCtx.java | 31 ++ .../entitiy/cf/CalculatedFieldState.java | 30 ++ .../cf/DefaultTbCalculatedFieldService.java | 281 ++++++++++++++++-- .../entitiy/cf/TbCalculatedFieldService.java | 8 +- .../CalculatedFieldControllerTest.java | 2 +- .../server/dao/asset/AssetService.java | 2 + .../server/dao/cf/CalculatedFieldService.java | 10 +- .../server/dao/device/DeviceService.java | 2 + .../server/dao/entity/EntityService.java | 3 + .../cf/BaseCalculatedFieldConfiguration.java | 158 ++++++++++ .../common/data/cf/CalculatedField.java | 8 +- .../data/cf/CalculatedFieldConfiguration.java | 48 +++ .../common/data/cf/CalculatedFieldLink.java | 4 +- .../cf/CalculatedFiledLinkConfiguration.java | 29 ++ .../SimpleCalculatedFieldConfiguration.java | 39 +++ common/proto/src/main/proto/queue.proto | 21 ++ .../server/dao/asset/AssetDao.java | 10 + .../server/dao/asset/BaseAssetService.java | 9 + .../dao/cf/BaseCalculatedFieldService.java | 60 ++-- .../dao/cf/CalculatedFieldConfigUtil.java | 172 +++++------ .../server/dao/cf/CalculatedFieldDao.java | 4 + .../server/dao/cf/CalculatedFieldLinkDao.java | 6 +- .../server/dao/device/DeviceDao.java | 12 + .../server/dao/device/DeviceServiceImpl.java | 9 + .../server/dao/entity/BaseEntityService.java | 5 + .../dao/model/sql/CalculatedFieldEntity.java | 18 +- .../model/sql/CalculatedFieldLinkEntity.java | 9 +- .../server/dao/sql/asset/AssetRepository.java | 10 + .../server/dao/sql/asset/JpaAssetDao.java | 11 + .../sql/cf/CalculatedFieldLinkRepository.java | 3 +- ...efaultNativeCalculatedFieldRepository.java | 146 +++++++++ .../dao/sql/cf/JpaCalculatedFieldDao.java | 11 + .../dao/sql/cf/JpaCalculatedFieldLinkDao.java | 14 +- .../cf/NativeCalculatedFieldRepository.java | 29 ++ .../dao/sql/device/DeviceRepository.java | 8 + .../server/dao/sql/device/JpaDeviceDao.java | 11 + .../server/dao/service/AssetServiceTest.java | 2 +- .../service/CalculatedFieldServiceTest.java | 2 +- .../dao/service/CustomerServiceTest.java | 2 +- .../server/dao/service/DeviceServiceTest.java | 2 +- 41 files changed, 1085 insertions(+), 159 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index 2b19d34e57..d2b3890c70 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -42,6 +42,7 @@ import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; @@ -99,6 +100,8 @@ public abstract class AbstractTbEntityService { protected AssetProfileService assetProfileService; @Autowired protected DeviceProfileService deviceProfileService; + @Autowired + protected EntityService entityService; protected boolean isTestProfile() { return Set.of(this.env.getActiveProfiles()).contains("test"); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java new file mode 100644 index 0000000000..61f28304f2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@Builder +public class CalculatedFieldCtx { + + private final CalculatedFieldId calculatedFieldId; + private final EntityId entityId; + private final CalculatedFieldState state; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java new file mode 100644 index 0000000000..dc07b820ea --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Builder; +import lombok.Data; + +import java.util.Map; + +@Data +@Builder +public class CalculatedFieldState { + + Map arguments; + String result; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index a7c77cf4bc..836f17650e 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -15,36 +15,64 @@ */ package org.thingsboard.server.service.entitiy.cf; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; +import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateEntityId; @@ -52,21 +80,213 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; @TbCoreComponent @Service @Slf4j +@RequiredArgsConstructor public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { - private final Map calculatedFields; - private final Map calculatedFieldLinks; private final CalculatedFieldService calculatedFieldService; + private final TbDeviceProfileCache deviceProfileCache; + private final TbAssetProfileCache assetProfileCache; + private final AttributesService attributesService; + private final TimeseriesService timeseriesService; + private ListeningScheduledExecutorService scheduledExecutor; - public DefaultTbCalculatedFieldService(CalculatedFieldService calculatedFieldService) { - this.calculatedFields = calculatedFieldService.findAll().stream().collect(Collectors.toMap(CalculatedField::getId, cf -> cf)); - this.calculatedFieldLinks = calculatedFieldService.findAllCalculatedFieldLinks().stream().collect(Collectors.toMap(CalculatedFieldLink::getId, cfl -> cfl)); - this.calculatedFieldService = calculatedFieldService; + private ListeningExecutorService calculatedFieldExecutor; + private ListeningExecutorService calculatedFieldCallbackExecutor; + + private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); + + @Value("${state.initFetchPackSize:50000}") + @Getter + private int initFetchPackSize; + + @Value("10") + @Getter + private int defaultCalculatedFieldCheckIntervalInSec; + + @PostConstruct + public void init() { + // from AbstractPartitionBasedService + scheduledExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("calculated-field-scheduled"))); + /// + calculatedFieldExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); + calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); + scheduledExecutor.scheduleWithFixedDelay(this::fetchCalculatedFields, new Random().nextInt(defaultCalculatedFieldCheckIntervalInSec), defaultCalculatedFieldCheckIntervalInSec, TimeUnit.SECONDS); + } + + @PreDestroy + public void stop() { + // from AbstractPartitionBasedService + if (scheduledExecutor != null) { + scheduledExecutor.shutdown(); + } + /// + if (calculatedFieldExecutor != null) { + calculatedFieldExecutor.shutdownNow(); + } + if (calculatedFieldCallbackExecutor != null) { + calculatedFieldCallbackExecutor.shutdownNow(); + } + } + + private ListenableFuture> fetchAttributesForEntity(TenantId tenantId, EntityId entityId, List keys) { + return attributesService.find(tenantId, entityId, AttributeScope.SERVER_SCOPE, keys); + } + + private ListenableFuture> fetchTimeSeries(TenantId tenantId, EntityId entityId, List keys) { + return timeseriesService.findLatest(tenantId, entityId, keys); + } + + private ListenableFuture initializeStateFromFutures(TenantId tenantId, EntityId entityId, CalculatedField calculatedField, List attributeKeys, List timeSeriesKeys) { + ListenableFuture> attributesFuture = fetchAttributesForEntity(tenantId, entityId, attributeKeys); + ListenableFuture> timeSeriesFuture = fetchTimeSeries(tenantId, entityId, timeSeriesKeys); + + ListenableFuture> combinedFuture = Futures.allAsList(attributesFuture, timeSeriesFuture); + + return Futures.transform(combinedFuture, results -> { + List attributes = (List) results.get(0); + List timeSeries = (List) results.get(1); + + initializeState(calculatedField, attributes, timeSeries); + + return null; + }, MoreExecutors.directExecutor()); + } + + private void initializeState(CalculatedField calculatedField, List attributes, List timeSeries) { + CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(calculatedField.getId(), + ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); + + CalculatedFieldState state = calculatedFieldCtx.getState(); + + if (state != null) { + String calculation = performCalculation(state.getArguments()); + + Map updatedArguments = state.getArguments(); + + state = CalculatedFieldState.builder() + .arguments(updatedArguments) + .result(calculation) + .build(); + } else { + // initial calculation + Map arguments = calculatedField.getConfiguration().getArguments(); + + Map argumentValues = arguments.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> resolveArgumentValue(entry.getKey(), entry.getValue(), attributes, timeSeries) + )); + + String calculation = performCalculation(argumentValues); + + state = CalculatedFieldState.builder() + .arguments(argumentValues) + .result(calculation) + .build(); + } + + calculatedFieldCtx = new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), state); + states.put(calculatedField.getId(), calculatedFieldCtx); + } + + private String resolveArgumentValue(String key, BaseCalculatedFieldConfiguration.Argument argument, + List attributes, List timeSeries) { + String type = argument.getType(); + String value = null; + + if ("ATTRIBUTES".equals(type)) { + value = attributes.stream() + .filter(attribute -> attribute.getKey().equals(key)) + .map(AttributeKvEntry::getValueAsString) + .findFirst() + .orElse(null); + } else if ("TIME_SERIES".equals(type)) { + value = timeSeries.stream() + .filter(tsKvEntry -> tsKvEntry.getKey().equals(key)) + .map(TsKvEntry::getValueAsString) + .findFirst() + .orElse(null); + } + + return value != null ? value : argument.getDefaultValue(); + } + + @Override + public void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); + List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); + if (cf != null) { + EntityId entityId = cf.getEntityId(); + calculatedFields.put(calculatedFieldId, cf); + calculatedFieldLinks.put(calculatedFieldId, links); + switch (entityId.getEntityType()) { + case ASSET, DEVICE: { + for (CalculatedFieldLink link : links) { + CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); + initializeStateFromFutures(tenantId, link.getEntityId(), cf, configuration.getAttributes(), configuration.getTimeSeries()); + } + } + case ASSET_PROFILE: { + PageDataIterable assetIds = new PageDataIterable<>(pageLink -> + assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); + for (AssetId assetId : assetIds) { + for (CalculatedFieldLink link : links) { + CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); + initializeStateFromFutures(tenantId, assetId, cf, configuration.getAttributes(), configuration.getTimeSeries()); + } + } + } + case DEVICE_PROFILE: { + PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> + deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); + for (DeviceId deviceId : deviceIds) { + for (CalculatedFieldLink link : links) { + CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); + initializeStateFromFutures(tenantId, deviceId, cf, configuration.getAttributes(), configuration.getTimeSeries()); + } + } + } + default: throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); + } + } else { + //Calculated field or entity was probably deleted while message was in queue; + callback.onSuccess(); + } + } catch (Exception e) { + log.trace("Failed to process calculated field add msg: [{}]", proto, e); + callback.onFailure(e); + } } @Override - public void onMsg() { + public void onCalculatedFieldUpdated(TransportProtos.CalculatedFieldUpdateMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + } catch (Exception e) { + log.trace("Failed to process calculated field update msg: [{}]", proto, e); + callback.onFailure(e); + } + } + @Override + public void onCalculatedFieldDeleted(TransportProtos.CalculatedFieldDeleteMsgProto proto, TbCallback callback) { + try { + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + calculatedFieldLinks.remove(calculatedFieldId); + calculatedFields.remove(calculatedFieldId); + states.remove(calculatedFieldId); + } catch (Exception e) { + log.trace("Failed to process calculated field delete msg: [{}]", proto, e); + callback.onFailure(e); + } } @Override @@ -105,28 +325,26 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } + private void fetchCalculatedFields() { + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + // TODO: read all states(CalculatedFieldCtx) + states.keySet().removeIf(calculatedFieldId -> !calculatedFields.containsKey(calculatedFieldId)); + } + private void checkEntityExistence(TenantId tenantId, EntityId entityId) { switch (entityId.getEntityType()) { - case ASSET -> Optional.ofNullable(assetService.findAssetById(tenantId, (AssetId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Asset with id [" + entityId.getId() + "] does not exist.")); - case DEVICE -> Optional.ofNullable(deviceService.findDeviceById(tenantId, (DeviceId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Device with id [" + entityId.getId() + "] does not exist.")); - case ASSET_PROFILE -> - Optional.ofNullable(assetProfileService.findAssetProfileById(tenantId, (AssetProfileId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Asset Profile with id [" + entityId.getId() + "] does not exist.")); - case DEVICE_PROFILE -> - Optional.ofNullable(deviceProfileService.findDeviceProfileById(tenantId, (DeviceProfileId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Device Profile with id [" + entityId.getId() + "] does not exist.")); + case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) + .orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); default -> throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); } } - private & HasTenantId, I extends EntityId> void checkReferencedEntities(CalculatedFieldConfig calculatedFieldConfig, SecurityUser user) throws ThingsboardException { - List referencedEntityIds = calculatedFieldConfig.getArguments().values().stream() - .map(CalculatedFieldConfig.Argument::getEntityId) - .filter(Objects::nonNull) - .toList(); + private & HasTenantId, I extends EntityId> void checkReferencedEntities(CalculatedFieldConfiguration calculatedFieldConfig, SecurityUser user) throws ThingsboardException { + List referencedEntityIds = calculatedFieldConfig.getReferencedEntities(); for (EntityId referencedEntityId : referencedEntityIds) { validateEntityId(referencedEntityId, id -> "Invalid entity id " + id); E entity = findEntity(user.getTenantId(), referencedEntityId); @@ -137,13 +355,14 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } private & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { - return (E) switch (entityId.getEntityType()) { - case TENANT -> tenantService.findTenantById((TenantId) entityId); - case CUSTOMER -> customerService.findCustomerById(tenantId, (CustomerId) entityId); - case ASSET -> assetService.findAssetById(tenantId, (AssetId) entityId); - case DEVICE -> deviceService.findDeviceById(tenantId, (DeviceId) entityId); + return switch (entityId.getEntityType()) { + case TENANT, CUSTOMER, ASSET, DEVICE -> (E) entityService.fetchEntity(tenantId, entityId).orElse(null); default -> throw new IllegalArgumentException("Calculated fields do not support entity type '" + entityId.getEntityType() + "' for referenced entities."); }; } + private String performCalculation(Map argumentValues) { + return "calculation"; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java index 2dc1cc35b2..4f14270c3d 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -18,11 +18,17 @@ package org.thingsboard.server.service.entitiy.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.security.model.SecurityUser; public interface TbCalculatedFieldService { - void onMsg(); + void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback); + + void onCalculatedFieldUpdated(TransportProtos.CalculatedFieldUpdateMsgProto proto, TbCallback callback); + + void onCalculatedFieldDeleted(TransportProtos.CalculatedFieldDeleteMsgProto proto, TbCallback callback); CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index affab6ee8b..3f505c78ef 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -126,7 +126,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); - calculatedField.setConfiguration(getCalculatedFieldConfig(null)); +// calculatedField.setConfiguration(getCalculatedFieldConfig(null)); calculatedField.setVersion(1L); return calculatedField; } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java index 7252db8097..2a9fe08827 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java @@ -63,6 +63,8 @@ public interface AssetService extends EntityDaoService { PageData findAssetInfosByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink); + PageData findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink); + ListenableFuture> findAssetsByTenantIdAndIdsAsync(TenantId tenantId, List assetIds); void deleteAssetsByTenantId(TenantId tenantId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 7760c49d2d..c12acade6c 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -21,6 +21,8 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.entity.EntityDaoService; import java.util.List; @@ -31,7 +33,9 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId); - List findAll(); + List findAllCalculatedFields(); + + PageData findAllCalculatedFields(PageLink pageLink); void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); @@ -43,6 +47,10 @@ public interface CalculatedFieldService extends EntityDaoService { List findAllCalculatedFieldLinks(); + List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + PageData findAllCalculatedFieldLinks(PageLink pageLink); + boolean existsByEntityId(TenantId tenantId, EntityId entityId); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java index 005c740571..0848e5a1cb 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java @@ -75,6 +75,8 @@ public interface DeviceService extends EntityDaoService { PageData findDevicesByTenantIdAndType(TenantId tenantId, String type, PageLink pageLink); + PageData findDeviceIdsByTenantIdAndDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink); + PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType type, PageLink pageLink); long countDevicesByTenantIdAndDeviceProfileIdAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType otaPackageType); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java index 5f522121d7..fc8859eceb 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.entity; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.NameLabelAndCustomerDetails; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -34,6 +35,8 @@ public interface EntityService { Optional fetchEntityCustomerId(TenantId tenantId, EntityId entityId); + Optional> fetchEntity(TenantId tenantId, EntityId entityId); + Optional fetchNameLabelAndCustomerDetails(TenantId tenantId, EntityId entityId); long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..9bb6d59428 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java @@ -0,0 +1,158 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + +@Data +public abstract class BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + @JsonIgnore + private final ObjectMapper mapper = new ObjectMapper(); + + protected Map arguments; + protected SimpleCalculatedFieldConfiguration.Output output; + + public BaseCalculatedFieldConfiguration() { + } + + public BaseCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + BaseCalculatedFieldConfiguration calculatedFieldConfig = toCalculatedFieldConfig(config, entityType, entityId); + this.arguments = calculatedFieldConfig.getArguments(); + this.output = calculatedFieldConfig.getOutput(); + } + + @Override + public List getReferencedEntities() { + return arguments.values().stream() + .map(SimpleCalculatedFieldConfiguration.Argument::getEntityId) + .collect(Collectors.toList()); + } + + @Override + public CalculatedFiledLinkConfiguration getReferencedEntityConfig(EntityId entityId) { + CalculatedFiledLinkConfiguration linkConfiguration = new CalculatedFiledLinkConfiguration(); + arguments.values().stream() + .filter(argument -> argument.getEntityId().equals(entityId)) + .forEach(argument -> { + switch (argument.getType()) { + case "ATTRIBUTES": + linkConfiguration.getAttributes().add(argument.getKey()); + break; + case "TIME_SERIES": + linkConfiguration.getTimeSeries().add(argument.getKey()); + break; + } + }); + + return linkConfiguration; + } + + @Override + public JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId) { + ObjectNode configNode = mapper.createObjectNode(); + + ObjectNode argumentsNode = configNode.putObject("arguments"); + arguments.forEach((key, argument) -> { + ObjectNode argumentNode = argumentsNode.putObject(key); + EntityId referencedEntityId = argument.getEntityId(); + if (referencedEntityId != null) { + argumentNode.put("entityType", referencedEntityId.getEntityType().name()); + argumentNode.put("entityId", referencedEntityId.getId().toString()); + } else { + argumentNode.put("entityType", entityType.name()); + argumentNode.put("entityId", entityId.toString()); + } + argumentNode.put("key", argument.getKey()); + argumentNode.put("type", argument.getType()); + argumentNode.put("defaultValue", argument.getDefaultValue()); + }); + + if (output != null) { + ObjectNode outputNode = configNode.putObject("output"); + outputNode.put("type", output.getType()); + outputNode.put("expression", output.getExpression()); + } + + return configNode; + } + + @Data + public static class Argument { + private EntityId entityId; + private String key; + private String type; + private String defaultValue; + } + + @Data + public static class Output { + private String type; + private String expression; + } + + private BaseCalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { + if (config == null || !config.isObject()) { + return null; + } + + Map arguments = new HashMap<>(); + JsonNode argumentsNode = config.get("arguments"); + if (argumentsNode != null && argumentsNode.isObject()) { + argumentsNode.fields().forEachRemaining(entry -> { + String key = entry.getKey(); + JsonNode argumentNode = entry.getValue(); + Argument argument = new Argument(); + if (argumentNode.hasNonNull("entityType") && argumentNode.hasNonNull("entityId")) { + String referencedEntityType = argumentNode.get("entityType").asText(); + UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); + argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); + } else { + argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + } + argument.setKey(argumentNode.get("key").asText()); + argument.setType(argumentNode.get("type").asText()); + argument.setDefaultValue(argumentNode.get("defaultValue").asText()); + arguments.put(key, argument); + }); + } + this.setArguments(arguments); + + JsonNode outputNode = config.get("output"); + if (outputNode != null) { + Output output = new Output(); + output.setType(outputNode.get("type").asText()); + output.setExpression(outputNode.get("expression").asText()); + this.setOutput(output); + } + + return this; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index 5c52ad0609..d96de37a39 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; +import java.io.Serializable; + @Schema @Data @EqualsAndHashCode(callSuper = true) @@ -46,8 +48,8 @@ public class CalculatedField extends BaseData implements HasN private String name; @Schema(description = "Version of calculated field configuration.", example = "0") private int configurationVersion; - @Schema - private transient CalculatedFieldConfig configuration; + @Schema(implementation = SimpleCalculatedFieldConfiguration.class) + private transient CalculatedFieldConfiguration configuration; @Getter @Setter private Long version; @@ -63,7 +65,7 @@ public class CalculatedField extends BaseData implements HasN super(id); } - public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, CalculatedFieldConfig configuration, Long version, CalculatedFieldId externalId) { + public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version, CalculatedFieldId externalId) { this.tenantId = tenantId; this.entityId = entityId; this.type = type; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java new file mode 100644 index 0000000000..e599b8c4d3 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.JsonNode; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE") +}) +public interface CalculatedFieldConfiguration { + + String getType(); + + Map getArguments(); + + List getReferencedEntities(); + + CalculatedFiledLinkConfiguration getReferencedEntityConfig(EntityId entityId); + + JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java index 2f176b13d2..922fda1f34 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java @@ -37,7 +37,7 @@ public class CalculatedFieldLink extends BaseData { @Schema(description = "JSON object with the Calculated Field Id. ", accessMode = Schema.AccessMode.READ_ONLY) private CalculatedFieldId calculatedFieldId; @Schema - private transient CalculatedFieldConfig configuration; + private transient CalculatedFiledLinkConfiguration configuration; public CalculatedFieldLink() { super(); @@ -47,7 +47,7 @@ public class CalculatedFieldLink extends BaseData { super(id); } - public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFieldConfig configuration) { + public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFiledLinkConfiguration configuration) { this.tenantId = tenantId; this.entityId = entityId; this.calculatedFieldId = calculatedFieldId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java new file mode 100644 index 0000000000..26d867fd7f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +public class CalculatedFiledLinkConfiguration { + + private List attributes = new ArrayList<>(); + private List timeSeries = new ArrayList<>(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..d635e0d82e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +@Data +public class SimpleCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + public SimpleCalculatedFieldConfiguration() { + super(); + } + + public SimpleCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + super(config, entityType, entityId); + } + + @Override + public String getType() { + return "SIMPLE"; + } +} diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 875a531566..e1d6320779 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1267,6 +1267,27 @@ message ToDeviceActorNotificationMsgProto { DeviceDeleteMsgProto deviceDeleteMsg = 8; } +message CalculatedFieldAddMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; +} + +message CalculatedFieldUpdateMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; +} + +message CalculatedFieldDeleteMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; +} + /** TB Core to Version Control Service */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index 42ab2e2545..8e40ec6d87 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -103,6 +103,16 @@ public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityD */ PageData findAssetInfosByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink); + /** + * Find asset ids by tenantId, assetProfileId and page link. + * + * @param tenantId the tenantId + * @param assetProfileId the assetProfileId + * @param pageLink the page link + * @return the list of asset objects + */ + PageData findAssetIdsByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink); + /** * Find assets by tenantId and assets Ids. * diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index f4c68aa15b..2135792174 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -285,6 +285,15 @@ public class BaseAssetService extends AbstractCachedEntityService findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink) { + log.trace("Executing findAssetIdsByTenantIdAndAssetProfileId, tenantId [{}], assetProfileId [{}]", tenantId, assetProfileId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(assetProfileId, id -> INCORRECT_ASSET_PROFILE_ID + id); + validatePageLink(pageLink); + return assetDao.findAssetIdsByTenantIdAndAssetProfileId(tenantId.getId(), assetProfileId.getId(), pageLink); + } + @Override public ListenableFuture> findAssetsByTenantIdAndIdsAsync(TenantId tenantId, List assetIds) { log.trace("Executing findAssetsByTenantIdAndIdsAsync, tenantId [{}], assetIds [{}]", tenantId, assetIds); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index ecc5aecce1..0210f6d70b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -20,20 +20,24 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import static org.thingsboard.server.dao.entity.AbstractEntityService.checkConstraintViolation; import static org.thingsboard.server.dao.service.Validator.validateId; +import static org.thingsboard.server.dao.service.Validator.validatePageLink; @Service("CalculatedFieldDaoService") @Slf4j @@ -74,11 +78,18 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { } @Override - public List findAll() { + public List findAllCalculatedFields() { log.trace("Executing findAll"); return calculatedFieldDao.findAll(); } + @Override + public PageData findAllCalculatedFields(PageLink pageLink) { + log.trace("Executing findAll, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return calculatedFieldDao.findAll(pageLink); + } + @Override public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); @@ -122,6 +133,19 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return calculatedFieldLinkDao.findAll(); } + @Override + public List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findAllCalculatedFieldLinksById, calculatedFieldId [{}]", calculatedFieldId); + return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId); + } + + @Override + public PageData findAllCalculatedFieldLinks(PageLink pageLink) { + log.trace("Executing findAllCalculatedFieldLinks, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return calculatedFieldLinkDao.findAll(pageLink); + } + @Override public boolean existsByEntityId(TenantId tenantId, EntityId entityId) { return calculatedFieldDao.existsByTenantIdAndEntityId(tenantId, entityId); @@ -132,9 +156,8 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return calculatedFieldDao.findAllByTenantId(tenantId).stream() .filter(calculatedField -> !referencedEntityId.equals(calculatedField.getEntityId())) .map(CalculatedField::getConfiguration) - .map(CalculatedFieldConfig::getArguments) - .flatMap(arguments -> arguments.values().stream()) - .anyMatch(argument -> referencedEntityId.equals(argument.getEntityId())); + .map(CalculatedFieldConfiguration::getReferencedEntities) + .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); } @Override @@ -148,21 +171,22 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { } private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { - CalculatedFieldLink existingLink = (calculatedField.getId() != null) - ? calculatedFieldLinkDao.findCalculatedFieldLinkByCalculatedFieldId(tenantId, calculatedField.getId()) - : null; - - CalculatedFieldLink updatedLink = buildCalculatedFieldLink(tenantId, calculatedField, existingLink); - saveCalculatedFieldLink(tenantId, updatedLink); + List links = buildCalculatedFieldLinks(tenantId, calculatedField); + links.forEach(link -> saveCalculatedFieldLink(tenantId, link)); } - private CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField, CalculatedFieldLink existingLink) { - CalculatedFieldLink link = (existingLink != null) ? existingLink : new CalculatedFieldLink(); - link.setTenantId(tenantId); - link.setEntityId(calculatedField.getEntityId()); - link.setCalculatedFieldId(calculatedField.getId()); - link.setConfiguration(calculatedField.getConfiguration()); - return link; + private List buildCalculatedFieldLinks(TenantId tenantId, CalculatedField calculatedField) { + CalculatedFieldConfiguration cfConfig = calculatedField.getConfiguration(); + return cfConfig.getReferencedEntities().stream() + .map(referencedEntityId -> { + CalculatedFieldLink link = new CalculatedFieldLink(); + link.setTenantId(tenantId); + link.setEntityId(referencedEntityId); + link.setCalculatedFieldId(calculatedField.getId()); + link.setConfiguration(cfConfig.getReferencedEntityConfig(referencedEntityId)); + return link; + }) + .collect(Collectors.toList()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java index 34dd885e12..980fff6273 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java @@ -19,7 +19,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -29,89 +31,89 @@ import java.util.UUID; public class CalculatedFieldConfigUtil { - public static CalculatedFieldConfig toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { - if (config == null) { - return null; - } - try { - CalculatedFieldConfig calculatedFieldConfig = new CalculatedFieldConfig(); - Map arguments = new HashMap<>(); - - JsonNode argumentsNode = config.get("arguments"); - if (argumentsNode != null && argumentsNode.isObject()) { - argumentsNode.fields().forEachRemaining(entry -> { - String key = entry.getKey(); - JsonNode argumentNode = entry.getValue(); - - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); - if (argumentNode.has("entityType") && argumentNode.has("entityId")) { - String referencedEntityType = argumentNode.get("entityType").asText(); - UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); - argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); - } else { - argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); - } - argument.setKey(argumentNode.get("key").asText()); - argument.setType(argumentNode.get("type").asText()); - - if (argumentNode.has("defaultValue")) { - argument.setDefaultValue(argumentNode.get("defaultValue").asInt()); - } - - arguments.put(key, argument); - }); - } - calculatedFieldConfig.setArguments(arguments); - - JsonNode outputNode = config.get("output"); - if (outputNode != null) { - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); - output.setType(outputNode.get("type").asText()); - output.setExpression(outputNode.get("expression").asText()); - calculatedFieldConfig.setOutput(output); - } - - return calculatedFieldConfig; - - } catch (Exception e) { - throw new IllegalArgumentException("Failed to convert JsonNode to CalculatedFieldConfig", e); - } - } - - public static JsonNode calculatedFieldConfigToJson(CalculatedFieldConfig calculatedFieldConfig, EntityType entityType, UUID entityId) { - if (calculatedFieldConfig == null) { - return null; - } - try { - ObjectNode configNode = JacksonUtil.newObjectNode(); - - ObjectNode argumentsNode = configNode.putObject("arguments"); - calculatedFieldConfig.getArguments().forEach((key, argument) -> { - ObjectNode argumentNode = argumentsNode.putObject(key); - EntityId referencedEntityId = argument.getEntityId(); - if (referencedEntityId != null) { - argumentNode.put("entityType", referencedEntityId.getEntityType().name()); - argumentNode.put("entityId", referencedEntityId.getId().toString()); - } else { - argumentNode.put("entityType", entityType.name()); - argumentNode.put("entityId", entityId.toString()); - } - argumentNode.put("key", argument.getKey()); - argumentNode.put("type", argument.getType()); - argumentNode.put("defaultValue", argument.getDefaultValue()); - }); - - if (calculatedFieldConfig.getOutput() != null) { - ObjectNode outputNode = configNode.putObject("output"); - outputNode.put("type", calculatedFieldConfig.getOutput().getType()); - outputNode.put("expression", calculatedFieldConfig.getOutput().getExpression()); - } - - return configNode; - - } catch (Exception e) { - throw new IllegalArgumentException("Failed to convert CalculatedFieldConfig to JsonNode", e); - } - } +// public static CalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { +// if (config == null) { +// return null; +// } +// try { +// CalculatedFieldConfiguration calculatedFieldConfig = new BaseCalculatedFieldConfiguration(); +// Map arguments = new HashMap<>(); +// +// JsonNode argumentsNode = config.get("arguments"); +// if (argumentsNode != null && argumentsNode.isObject()) { +// argumentsNode.fields().forEachRemaining(entry -> { +// String key = entry.getKey(); +// JsonNode argumentNode = entry.getValue(); +// +// CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); +// if (argumentNode.has("entityType") && argumentNode.has("entityId")) { +// String referencedEntityType = argumentNode.get("entityType").asText(); +// UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); +// argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); +// } else { +// argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); +// } +// argument.setKey(argumentNode.get("key").asText()); +// argument.setType(argumentNode.get("type").asText()); +// +// if (argumentNode.has("defaultValue")) { +// argument.setDefaultValue(argumentNode.get("defaultValue").asInt()); +// } +// +// arguments.put(key, argument); +// }); +// } +// calculatedFieldConfig.setArguments(arguments); +// +// JsonNode outputNode = config.get("output"); +// if (outputNode != null) { +// CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); +// output.setType(outputNode.get("type").asText()); +// output.setExpression(outputNode.get("expression").asText()); +// calculatedFieldConfig.setOutput(output); +// } +// +// return calculatedFieldConfig; +// +// } catch (Exception e) { +// throw new IllegalArgumentException("Failed to convert JsonNode to CalculatedFieldConfig", e); +// } +// } +// +// public static JsonNode calculatedFieldConfigToJson(CalculatedFieldConfiguration calculatedFieldConfig, EntityType entityType, UUID entityId) { +// if (calculatedFieldConfig == null) { +// return null; +// } +// try { +// ObjectNode configNode = JacksonUtil.newObjectNode(); +// +// ObjectNode argumentsNode = configNode.putObject("arguments"); +// calculatedFieldConfig.getArguments().forEach((key, argument) -> { +// ObjectNode argumentNode = argumentsNode.putObject(key); +// EntityId referencedEntityId = argument.getEntityId(); +// if (referencedEntityId != null) { +// argumentNode.put("entityType", referencedEntityId.getEntityType().name()); +// argumentNode.put("entityId", referencedEntityId.getId().toString()); +// } else { +// argumentNode.put("entityType", entityType.name()); +// argumentNode.put("entityId", entityId.toString()); +// } +// argumentNode.put("key", argument.getKey()); +// argumentNode.put("type", argument.getType()); +// argumentNode.put("defaultValue", argument.getDefaultValue()); +// }); +// +// if (calculatedFieldConfig.getOutput() != null) { +// ObjectNode outputNode = configNode.putObject("output"); +// outputNode.put("type", calculatedFieldConfig.getOutput().getType()); +// outputNode.put("expression", calculatedFieldConfig.getOutput().getExpression()); +// } +// +// return configNode; +// +// } catch (Exception e) { +// throw new IllegalArgumentException("Failed to convert CalculatedFieldConfig to JsonNode", e); +// } +// } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index bb52d0e2c2..4abe02a09b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -18,6 +18,8 @@ package org.thingsboard.server.dao.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; import java.util.List; @@ -30,6 +32,8 @@ public interface CalculatedFieldDao extends Dao { List findAll(); + PageData findAll(PageLink pageLink); + List removeAllByEntityId(TenantId tenantId, EntityId entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index c4a06c88a3..728e19b890 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -18,14 +18,18 @@ package org.thingsboard.server.dao.cf; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; import java.util.List; public interface CalculatedFieldLinkDao extends Dao { - CalculatedFieldLink findCalculatedFieldLinkByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); List findAll(); + PageData findAll(PageLink pageLink); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 2305e4419c..7c82df7703 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -22,7 +22,9 @@ import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; @@ -85,6 +87,16 @@ public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntit */ PageData findDevicesByTenantIdAndType(UUID tenantId, String type, PageLink pageLink); + /** + * Find device ids by tenantId, type and page link. + * + * @param tenantId the tenantId + * @param deviceProfileId the deviceProfileId + * @param pageLink the page link + * @return the list of device objects + */ + PageData findDeviceIdsByTenantIdAndDeviceProfileId(UUID tenantId, UUID deviceProfileId, PageLink pageLink); + PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(UUID tenantId, UUID deviceProfileId, OtaPackageType type, diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index ac6b8330a9..ffebd5f032 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -395,6 +395,15 @@ public class DeviceServiceImpl extends CachedVersionedEntityService findDeviceIdsByTenantIdAndDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink) { + log.trace("Executing findDeviceIdsByTenantIdAndType, tenantId [{}], deviceProfileId [{}], pageLink [{}]", tenantId, deviceProfileId, pageLink); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(deviceProfileId, id -> INCORRECT_DEVICE_PROFILE_ID + id); + validatePageLink(pageLink); + return deviceDao.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId.getId(), deviceProfileId.getId(), pageLink); + } + @Override public PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 3cdcf5e874..5efea9a6df 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -134,6 +134,11 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe return fetchAndConvert(tenantId, entityId, this::getNameLabelAndCustomerDetails); } + @Override + public Optional> fetchEntity(TenantId tenantId, EntityId entityId) { + return fetchAndConvert(tenantId, entityId, Function.identity()); + } + private Optional fetchAndConvert(TenantId tenantId, EntityId entityId, Function, T> converter) { EntityDaoService entityDaoService = entityServiceRegistry.getServiceByEntityType(entityId.getEntityType()); Optional> entityOpt = entityDaoService.findEntity(tenantId, entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 98eb050d28..9cb4f08640 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -24,6 +24,8 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -33,8 +35,6 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.toCalculatedFieldConfig; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION_VERSION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_ID; @@ -93,7 +93,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.type = calculatedField.getType(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); - this.configuration = calculatedFieldConfigToJson(calculatedField.getConfiguration(), entityType, entityId); + this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(entityType, entityId); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { this.externalId = calculatedField.getExternalId().getId(); @@ -109,7 +109,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setType(type); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(toCalculatedFieldConfig(configuration, entityType, entityId)); + calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, entityType, entityId)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); @@ -117,4 +117,14 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem return calculatedField; } + private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + String type = config.get("type").asText(); + switch (type) { + case "SIMPLE": + return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + default: + throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); + } + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 73a2f9fbdf..3dc08d6bf1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -22,8 +22,10 @@ import jakarta.persistence.Entity; import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -34,8 +36,6 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.toCalculatedFieldConfig; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_ID; @@ -65,7 +65,6 @@ public class CalculatedFieldLinkEntity extends BaseSqlEntity, Expor @Param("textSearch") String textSearch, Pageable pageable); + @Query("SELECT a.id FROM AssetEntity a " + + "WHERE a.tenantId = :tenantId " + + "AND a.assetProfileId = :assetProfileId " + + "AND (:textSearch IS NULL OR ilike(a.type, CONCAT('%', :textSearch, '%')) = true) ") + Page findAssetIdsByTenantIdAndAssetProfileId(@Param("tenantId") UUID tenantId, + @Param("assetProfileId") UUID assetProfileId, + @Param("textSearch") String textSearch, + Pageable pageable); + @Query("SELECT a FROM AssetEntity a WHERE a.tenantId = :tenantId " + "AND a.customerId = :customerId AND a.type = :type " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 0ef4370f30..db6472f825 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -41,6 +41,7 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; import static org.thingsboard.server.dao.DaoUtil.convertTenantEntityInfosToDto; @@ -159,6 +160,16 @@ public class JpaAssetDao extends JpaAbstractDao implements A DaoUtil.toPageable(pageLink, AssetInfoEntity.assetInfoColumnMap))); } + @Override + public PageData findAssetIdsByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink) { + return DaoUtil.pageToPageData(assetRepository.findAssetIdsByTenantIdAndAssetProfileId( + tenantId, + assetProfileId, + pageLink.getTextSearch(), + DaoUtil.toPageable(pageLink))) + .mapData(AssetId::new); + } + @Override public PageData findAssetsByTenantIdAndCustomerIdAndType(UUID tenantId, UUID customerId, String type, PageLink pageLink) { return DaoUtil.toPageData(assetRepository diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java index fa1c5ce38e..61c4026cca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java @@ -18,10 +18,11 @@ package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; +import java.util.List; import java.util.UUID; public interface CalculatedFieldLinkRepository extends JpaRepository { - CalculatedFieldLinkEntity findByTenantIdAndCalculatedFieldId(UUID tenantId, UUID calculatedFieldId); + List findAllByTenantIdAndCalculatedFieldId(UUID tenantId, UUID calculatedFieldId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java new file mode 100644 index 0000000000..8401d99128 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -0,0 +1,146 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + +@RequiredArgsConstructor +@Repository +@Slf4j +public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedFieldRepository { + + private final String CF_COUNT_QUERY = "SELECT count(id) FROM calculated_field;"; + private final String CF_QUERY = "SELECT * FROM calculated_field ORDER BY created_time ASC LIMIT %s OFFSET %s"; + + private final String CFL_COUNT_QUERY = "SELECT count(id) FROM calculated_field_link;"; + private final String CFL_QUERY = "SELECT * FROM calculated_field_link ORDER BY created_time ASC LIMIT %s OFFSET %s"; + + private final NamedParameterJdbcTemplate jdbcTemplate; + private final TransactionTemplate transactionTemplate; + + @Override + public PageData findCalculatedFields(Pageable pageable) { + return transactionTemplate.execute(status -> { + long startTs = System.currentTimeMillis(); + int totalElements = jdbcTemplate.queryForObject(CF_COUNT_QUERY, Collections.emptyMap(), Integer.class); + log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); + startTs = System.currentTimeMillis(); + List> rows = jdbcTemplate.queryForList(String.format(CF_QUERY, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); + log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); + int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; + boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); + var data = rows.stream().map(row -> { + + UUID id = (UUID) row.get("id"); + long createdTime = (long) row.get("created_time"); + UUID tenantId = (UUID) row.get("tenant_id"); + EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); + UUID entityId = (UUID) row.get("entity_id"); + String type = (String) row.get("type"); + String name = (String) row.get("name"); + int configurationVersion = (int) row.get("configuration_version"); + JsonNode configuration = JacksonUtil.valueToTree(row.get("configuration")); + long version = (long) row.get("version"); + Object externalIdObj = row.get("external_id"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setId(new CalculatedFieldId(id)); + calculatedField.setCreatedTime(createdTime); + calculatedField.setTenantId(new TenantId(tenantId)); + calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedField.setType(type); + calculatedField.setName(name); + calculatedField.setConfigurationVersion(configurationVersion); + calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, entityType, entityId)); + calculatedField.setVersion(version); + calculatedField.setExternalId(externalIdObj != null ? new CalculatedFieldId(UUID.fromString((String) externalIdObj)) : null); + + return calculatedField; + }).collect(Collectors.toList()); + return new PageData<>(data, totalPages, totalElements, hasNext); + }); + } + + @Override + public PageData findCalculatedFieldLinks(Pageable pageable) { + return transactionTemplate.execute(status -> { + long startTs = System.currentTimeMillis(); + int totalElements = jdbcTemplate.queryForObject(CFL_COUNT_QUERY, Collections.emptyMap(), Integer.class); + log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); + startTs = System.currentTimeMillis(); + List> rows = jdbcTemplate.queryForList(String.format(CFL_QUERY, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); + log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); + int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; + boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); + var data = rows.stream().map(row -> { + + UUID id = (UUID) row.get("id"); + long createdTime = (long) row.get("created_time"); + UUID tenantId = (UUID) row.get("tenant_id"); + EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); + UUID entityId = (UUID) row.get("entity_id"); + UUID calculatedFieldId = (UUID) row.get("calculated_field_id"); + JsonNode configuration = JacksonUtil.valueToTree(row.get("configuration")); + + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); + calculatedFieldLink.setId(new CalculatedFieldLinkId(id)); + calculatedFieldLink.setCreatedTime(createdTime); + calculatedFieldLink.setTenantId(new TenantId(tenantId)); + calculatedFieldLink.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedFieldLink.setCalculatedFieldId(new CalculatedFieldId(calculatedFieldId)); + calculatedFieldLink.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFiledLinkConfiguration.class)); + + return calculatedFieldLink; + }).collect(Collectors.toList()); + return new PageData<>(data, totalPages, totalElements, hasNext); + }); + } + + private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + String type = config.get("type").asText(); + switch (type) { + case "SIMPLE": + return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + default: + throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); + } + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 4bf4c1ead7..1137b91947 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -18,16 +18,20 @@ package org.thingsboard.server.dao.sql.cf; import jakarta.transaction.Transactional; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.sql.device.NativeDeviceRepository; import org.thingsboard.server.dao.util.SqlDao; import java.util.List; @@ -40,6 +44,7 @@ import java.util.UUID; public class JpaCalculatedFieldDao extends JpaAbstractDao implements CalculatedFieldDao { private final CalculatedFieldRepository calculatedFieldRepository; + private final NativeCalculatedFieldRepository nativeCalculatedFieldRepository; @Override public boolean existsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId) { @@ -56,6 +61,12 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAll(PageLink pageLink) { + log.debug("Try to find calculated fields by pageLink [{}]", pageLink); + return nativeCalculatedFieldRepository.findCalculatedFields(DaoUtil.toPageable(pageLink)); + } + @Override @Transactional public List removeAllByEntityId(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index f584a8d76c..a2f8f224c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -22,7 +22,10 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; @@ -39,10 +42,11 @@ import java.util.UUID; public class JpaCalculatedFieldLinkDao extends JpaAbstractDao implements CalculatedFieldLinkDao { private final CalculatedFieldLinkRepository calculatedFieldLinkRepository; + private final NativeCalculatedFieldRepository nativeCalculatedFieldRepository; @Override - public CalculatedFieldLink findCalculatedFieldLinkByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - return DaoUtil.getData(calculatedFieldLinkRepository.findByTenantIdAndCalculatedFieldId(tenantId.getId(), calculatedFieldId.getId())); + public List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAllByTenantIdAndCalculatedFieldId(tenantId.getId(), calculatedFieldId.getId())); } @Override @@ -50,6 +54,12 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao findAll(PageLink pageLink) { + log.debug("Try to find calculated field links by pageLink [{}]", pageLink); + return nativeCalculatedFieldRepository.findCalculatedFieldLinks(DaoUtil.toPageable(pageLink)); + } + @Override protected Class getEntityClass() { return CalculatedFieldLinkEntity.class; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java new file mode 100644 index 0000000000..76fed8c311 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import org.springframework.data.domain.Pageable; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.page.PageData; + +public interface NativeCalculatedFieldRepository { + + PageData findCalculatedFields(Pageable pageable); + + PageData findCalculatedFieldLinks(Pageable pageable); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java index c55210b606..5093da78c5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java @@ -81,6 +81,14 @@ public interface DeviceRepository extends JpaRepository, Exp @Param("textSearch") String textSearch, Pageable pageable); + @Query("SELECT d.id FROM DeviceEntity d WHERE d.tenantId = :tenantId " + + "AND d.deviceProfileId = :deviceProfileId " + + "AND (:textSearch IS NULL OR ilike(d.type, CONCAT('%', :textSearch, '%')) = true)") + Page findIdsByTenantIdAndDeviceProfileId(@Param("tenantId") UUID tenantId, + @Param("deviceProfileId") UUID deviceProfileId, + @Param("textSearch") String textSearch, + Pageable pageable); + @Query("SELECT d FROM DeviceEntity d WHERE d.tenantId = :tenantId " + "AND d.deviceProfileId = :deviceProfileId " + "AND d.firmwareId IS NULL") diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 48bb998016..bf0def9fca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -173,6 +173,17 @@ public class JpaDeviceDao extends JpaAbstractDao implement DaoUtil.toPageable(pageLink))); } + @Override + public PageData findDeviceIdsByTenantIdAndDeviceProfileId(UUID tenantId, UUID deviceProfileId, PageLink pageLink) { + return DaoUtil.pageToPageData( + deviceRepository.findIdsByTenantIdAndDeviceProfileId( + tenantId, + deviceProfileId, + pageLink.getTextSearch(), + DaoUtil.toPageable(pageLink))) + .mapData(DeviceId::new); + } + @Override public PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(UUID tenantId, UUID deviceProfileId, diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index ba0c73fb09..311bf23263 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -892,7 +892,7 @@ public class AssetServiceTest extends AbstractServiceTest { config.setOutput(output); - calculatedField.setConfiguration(config); +// calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index c4ad2856a9..8e50e1e0fb 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -151,7 +151,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); - calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); +// calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); calculatedField.setVersion(1L); return calculatedField; } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 548a5bfcf5..3f990b9dd7 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -387,7 +387,7 @@ public class CustomerServiceTest extends AbstractServiceTest { config.setOutput(output); - calculatedField.setConfiguration(config); +// calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 16bb08350b..90d04d5216 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -1230,7 +1230,7 @@ public class DeviceServiceTest extends AbstractServiceTest { config.setOutput(output); - calculatedField.setConfiguration(config); +// calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); From 444c5bf07976df4e8ac7d57fb870a120dd673551 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 12 Nov 2024 12:25:59 +0200 Subject: [PATCH 02/13] refactored service and test classes --- .../cf/DefaultTbCalculatedFieldService.java | 48 ++++--- .../cf/BaseCalculatedFieldConfiguration.java | 12 +- .../common/data/cf/CalculatedFieldConfig.java | 43 ------- .../data/cf/CalculatedFieldConfiguration.java | 7 +- .../common/data/cf/CalculatedFieldLink.java | 4 +- ... => CalculatedFieldLinkConfiguration.java} | 2 +- .../dao/cf/BaseCalculatedFieldService.java | 9 +- .../dao/cf/CalculatedFieldConfigUtil.java | 119 ------------------ .../dao/model/sql/CalculatedFieldEntity.java | 3 +- .../model/sql/CalculatedFieldLinkEntity.java | 4 +- ...efaultNativeCalculatedFieldRepository.java | 9 +- .../main/resources/sql/schema-entities.sql | 4 +- .../server/dao/service/AssetServiceTest.java | 12 +- .../service/CalculatedFieldServiceTest.java | 36 +++--- .../dao/service/CustomerServiceTest.java | 12 +- .../server/dao/service/DeviceServiceTest.java | 13 +- 16 files changed, 84 insertions(+), 253 deletions(-) delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{CalculatedFiledLinkConfiguration.java => CalculatedFieldLinkConfiguration.java} (94%) delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 836f17650e..e222259a53 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -38,7 +38,7 @@ import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -58,8 +58,6 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; -import org.thingsboard.server.service.profile.TbAssetProfileCache; -import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; @@ -84,8 +82,6 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { private final CalculatedFieldService calculatedFieldService; - private final TbDeviceProfileCache deviceProfileCache; - private final TbAssetProfileCache assetProfileCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; private ListeningScheduledExecutorService scheduledExecutor; @@ -215,6 +211,15 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp return value != null ? value : argument.getDefaultValue(); } + private void initializeForProfile(TenantId tenantId, List links, CalculatedField cf, Iterable profileIds) { + for (T profileId : profileIds) { + for (CalculatedFieldLink link : links) { + CalculatedFieldLinkConfiguration configuration = link.getConfiguration(); + initializeStateFromFutures(tenantId, profileId, cf, configuration.getAttributes(), configuration.getTimeSeries()); + } + } + } + @Override public void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback) { try { @@ -227,33 +232,24 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); switch (entityId.getEntityType()) { - case ASSET, DEVICE: { + case ASSET, DEVICE -> { for (CalculatedFieldLink link : links) { - CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); + CalculatedFieldLinkConfiguration configuration = link.getConfiguration(); initializeStateFromFutures(tenantId, link.getEntityId(), cf, configuration.getAttributes(), configuration.getTimeSeries()); } } - case ASSET_PROFILE: { + case ASSET_PROFILE -> { PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - for (AssetId assetId : assetIds) { - for (CalculatedFieldLink link : links) { - CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); - initializeStateFromFutures(tenantId, assetId, cf, configuration.getAttributes(), configuration.getTimeSeries()); - } - } + initializeForProfile(tenantId, links, cf, assetIds); } - case DEVICE_PROFILE: { + case DEVICE_PROFILE -> { PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - for (DeviceId deviceId : deviceIds) { - for (CalculatedFieldLink link : links) { - CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); - initializeStateFromFutures(tenantId, deviceId, cf, configuration.getAttributes(), configuration.getTimeSeries()); - } - } + initializeForProfile(tenantId, links, cf, deviceIds); } - default: throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); + default -> + throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); } } else { //Calculated field or entity was probably deleted while message was in queue; @@ -336,8 +332,9 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private void checkEntityExistence(TenantId tenantId, EntityId entityId) { switch (entityId.getEntityType()) { - case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) - .orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); + case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> + Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) + .orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); default -> throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); } @@ -357,7 +354,8 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { case TENANT, CUSTOMER, ASSET, DEVICE -> (E) entityService.fetchEntity(tenantId, entityId).orElse(null); - default -> throw new IllegalArgumentException("Calculated fields do not support entity type '" + entityId.getEntityType() + "' for referenced entities."); + default -> + throw new IllegalArgumentException("Calculated fields do not support entity type '" + entityId.getEntityType() + "' for referenced entities."); }; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java index 9bb6d59428..3894835abc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.UUID; import java.util.stream.Collectors; @@ -37,7 +38,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel private final ObjectMapper mapper = new ObjectMapper(); protected Map arguments; - protected SimpleCalculatedFieldConfiguration.Output output; + protected Output output; public BaseCalculatedFieldConfiguration() { } @@ -45,19 +46,20 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel public BaseCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { BaseCalculatedFieldConfiguration calculatedFieldConfig = toCalculatedFieldConfig(config, entityType, entityId); this.arguments = calculatedFieldConfig.getArguments(); - this.output = calculatedFieldConfig.getOutput(); + this.output = calculatedFieldConfig.getOutput(); } @Override public List getReferencedEntities() { return arguments.values().stream() - .map(SimpleCalculatedFieldConfiguration.Argument::getEntityId) + .map(Argument::getEntityId) + .filter(Objects::nonNull) .collect(Collectors.toList()); } @Override - public CalculatedFiledLinkConfiguration getReferencedEntityConfig(EntityId entityId) { - CalculatedFiledLinkConfiguration linkConfiguration = new CalculatedFiledLinkConfiguration(); + public CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId) { + CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); arguments.values().stream() .filter(argument -> argument.getEntityId().equals(entityId)) .forEach(argument -> { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java deleted file mode 100644 index b51258b2ec..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.cf; - -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; - -import java.util.Map; - -@Data -public class CalculatedFieldConfig { - - private Map arguments; - private Output output; - - @Data - public static class Argument { - private EntityId entityId; - private String key; - private String type; - private int defaultValue; - } - - @Data - public static class Output { - private String type; - private String expression; - } - -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index e599b8c4d3..deaeebcb50 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.cf; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.JsonNode; @@ -35,14 +36,18 @@ import java.util.UUID; }) public interface CalculatedFieldConfiguration { + @JsonIgnore String getType(); Map getArguments(); + @JsonIgnore List getReferencedEntities(); - CalculatedFiledLinkConfiguration getReferencedEntityConfig(EntityId entityId); + @JsonIgnore + CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId); + @JsonIgnore JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java index 922fda1f34..f9f9ee625f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java @@ -37,7 +37,7 @@ public class CalculatedFieldLink extends BaseData { @Schema(description = "JSON object with the Calculated Field Id. ", accessMode = Schema.AccessMode.READ_ONLY) private CalculatedFieldId calculatedFieldId; @Schema - private transient CalculatedFiledLinkConfiguration configuration; + private transient CalculatedFieldLinkConfiguration configuration; public CalculatedFieldLink() { super(); @@ -47,7 +47,7 @@ public class CalculatedFieldLink extends BaseData { super(id); } - public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFiledLinkConfiguration configuration) { + public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFieldLinkConfiguration configuration) { this.tenantId = tenantId; this.entityId = entityId; this.calculatedFieldId = calculatedFieldId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java similarity index 94% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java index 26d867fd7f..02d668a67a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java @@ -21,7 +21,7 @@ import java.util.ArrayList; import java.util.List; @Data -public class CalculatedFiledLinkConfiguration { +public class CalculatedFieldLinkConfiguration { private List attributes = new ArrayList<>(); private List timeSeries = new ArrayList<>(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 0210f6d70b..db5539eb96 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -110,13 +110,8 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { @Override public CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink) { calculatedFieldLinkDataValidator.validate(calculatedFieldLink, CalculatedFieldLink::getTenantId); - try { - log.trace("Executing save calculated field link, [{}]", calculatedFieldLink); - return calculatedFieldLinkDao.save(tenantId, calculatedFieldLink); - } catch (Exception e) { - checkConstraintViolation(e, "calculated_field_link_unq_key", "Calculated Field for such entity id is already exists!"); - throw e; - } + log.trace("Executing save calculated field link, [{}]", calculatedFieldLink); + return calculatedFieldLinkDao.save(tenantId, calculatedFieldLink); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java deleted file mode 100644 index 980fff6273..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.cf; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; - -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -public class CalculatedFieldConfigUtil { - -// public static CalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { -// if (config == null) { -// return null; -// } -// try { -// CalculatedFieldConfiguration calculatedFieldConfig = new BaseCalculatedFieldConfiguration(); -// Map arguments = new HashMap<>(); -// -// JsonNode argumentsNode = config.get("arguments"); -// if (argumentsNode != null && argumentsNode.isObject()) { -// argumentsNode.fields().forEachRemaining(entry -> { -// String key = entry.getKey(); -// JsonNode argumentNode = entry.getValue(); -// -// CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); -// if (argumentNode.has("entityType") && argumentNode.has("entityId")) { -// String referencedEntityType = argumentNode.get("entityType").asText(); -// UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); -// argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); -// } else { -// argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); -// } -// argument.setKey(argumentNode.get("key").asText()); -// argument.setType(argumentNode.get("type").asText()); -// -// if (argumentNode.has("defaultValue")) { -// argument.setDefaultValue(argumentNode.get("defaultValue").asInt()); -// } -// -// arguments.put(key, argument); -// }); -// } -// calculatedFieldConfig.setArguments(arguments); -// -// JsonNode outputNode = config.get("output"); -// if (outputNode != null) { -// CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); -// output.setType(outputNode.get("type").asText()); -// output.setExpression(outputNode.get("expression").asText()); -// calculatedFieldConfig.setOutput(output); -// } -// -// return calculatedFieldConfig; -// -// } catch (Exception e) { -// throw new IllegalArgumentException("Failed to convert JsonNode to CalculatedFieldConfig", e); -// } -// } -// -// public static JsonNode calculatedFieldConfigToJson(CalculatedFieldConfiguration calculatedFieldConfig, EntityType entityType, UUID entityId) { -// if (calculatedFieldConfig == null) { -// return null; -// } -// try { -// ObjectNode configNode = JacksonUtil.newObjectNode(); -// -// ObjectNode argumentsNode = configNode.putObject("arguments"); -// calculatedFieldConfig.getArguments().forEach((key, argument) -> { -// ObjectNode argumentNode = argumentsNode.putObject(key); -// EntityId referencedEntityId = argument.getEntityId(); -// if (referencedEntityId != null) { -// argumentNode.put("entityType", referencedEntityId.getEntityType().name()); -// argumentNode.put("entityId", referencedEntityId.getId().toString()); -// } else { -// argumentNode.put("entityType", entityType.name()); -// argumentNode.put("entityId", entityId.toString()); -// } -// argumentNode.put("key", argument.getKey()); -// argumentNode.put("type", argument.getType()); -// argumentNode.put("defaultValue", argument.getDefaultValue()); -// }); -// -// if (calculatedFieldConfig.getOutput() != null) { -// ObjectNode outputNode = configNode.putObject("output"); -// outputNode.put("type", calculatedFieldConfig.getOutput().getType()); -// outputNode.put("expression", calculatedFieldConfig.getOutput().getExpression()); -// } -// -// return configNode; -// -// } catch (Exception e) { -// throw new IllegalArgumentException("Failed to convert CalculatedFieldConfig to JsonNode", e); -// } -// } - -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 9cb4f08640..725ddd7bef 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -117,8 +117,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem return calculatedField; } - private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - String type = config.get("type").asText(); + private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { switch (type) { case "SIMPLE": return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 3dc08d6bf1..0ac5a9bc94 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -25,7 +25,7 @@ import lombok.EqualsAndHashCode; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -86,7 +86,7 @@ public class CalculatedFieldLinkEntity extends BaseSqlEntity calculatedFieldService.saveCalculatedFieldLink(tenantId, calculatedFieldLink)) - .isInstanceOf(DataValidationException.class) - .hasMessage("Calculated Field for such entity id is already exists!"); - } - private CalculatedField saveValidCalculatedField() { Device device = createTestDevice(); CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); @@ -148,10 +140,10 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setEntityId(entityId); - calculatedField.setType("Simple"); + calculatedField.setType("SIMPLE"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); -// calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); + calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); calculatedField.setVersion(1L); return calculatedField; } @@ -160,22 +152,28 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); calculatedFieldLink.setTenantId(tenantId); calculatedFieldLink.setEntityId(calculatedField.getEntityId()); -// calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); + calculatedFieldLink.setConfiguration(getCalculatedFieldLinkConfiguration()); calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); return calculatedFieldLink; } - private CalculatedFieldConfig getCalculatedFieldConfig(EntityId referencedEntityId) { - CalculatedFieldConfig config = new CalculatedFieldConfig(); + private CalculatedFieldLinkConfiguration getCalculatedFieldLinkConfiguration() { + CalculatedFieldLinkConfiguration calculatedFieldLinkConfiguration = new CalculatedFieldLinkConfiguration(); + calculatedFieldLinkConfiguration.setTimeSeries(List.of("temperature")); + return calculatedFieldLinkConfiguration; + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(referencedEntityId); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 3f990b9dd7..0e2d9b797b 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -32,7 +32,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -369,25 +369,25 @@ public class CustomerServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("Simple"); + calculatedField.setType("SIMPLE"); calculatedField.setEntityId(savedAsset.getId()); - CalculatedFieldConfig config = new CalculatedFieldConfig(); + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(savedCustomer.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); config.setOutput(output); -// calculatedField.setConfiguration(config); + calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 90d04d5216..33b80c0d99 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -40,7 +40,7 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -1212,25 +1212,25 @@ public class DeviceServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("Simple"); + calculatedField.setType("SIMPLE"); calculatedField.setEntityId(deviceWithCf.getId()); - CalculatedFieldConfig config = new CalculatedFieldConfig(); + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(device.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); config.setOutput(output); -// calculatedField.setConfiguration(config); + calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); @@ -1241,5 +1241,4 @@ public class DeviceServiceTest extends AbstractServiceTest { calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); } - } From b6b7af8003d7737c8cc5f818c85711b338329db9 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 12 Nov 2024 12:51:29 +0200 Subject: [PATCH 03/13] refactored tests --- .../CalculatedFieldControllerTest.java | 15 ++++++++------- .../service/CalculatedFieldServiceTest.java | 18 ------------------ 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 3f505c78ef..67e7bc64bd 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -22,7 +22,8 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.security.Authority; @@ -123,25 +124,25 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { private CalculatedField getCalculatedField(DeviceId deviceId) { CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(deviceId); - calculatedField.setType("Simple"); + calculatedField.setType("SIMPLE"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); -// calculatedField.setConfiguration(getCalculatedFieldConfig(null)); + calculatedField.setConfiguration(getCalculatedFieldConfig(null)); calculatedField.setVersion(1L); return calculatedField; } - private CalculatedFieldConfig getCalculatedFieldConfig(EntityId referencedEntityId) { - CalculatedFieldConfig config = new CalculatedFieldConfig(); + private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(referencedEntityId); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index de6836900d..a94eb83fa3 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -25,8 +25,6 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -34,7 +32,6 @@ import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; -import java.util.List; import java.util.Map; import java.util.UUID; @@ -148,21 +145,6 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { return calculatedField; } - private CalculatedFieldLink getCalculatedFieldLink(CalculatedField calculatedField) { - CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); - calculatedFieldLink.setTenantId(tenantId); - calculatedFieldLink.setEntityId(calculatedField.getEntityId()); - calculatedFieldLink.setConfiguration(getCalculatedFieldLinkConfiguration()); - calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); - return calculatedFieldLink; - } - - private CalculatedFieldLinkConfiguration getCalculatedFieldLinkConfiguration() { - CalculatedFieldLinkConfiguration calculatedFieldLinkConfiguration = new CalculatedFieldLinkConfiguration(); - calculatedFieldLinkConfiguration.setTimeSeries(List.of("temperature")); - return calculatedFieldLinkConfiguration; - } - private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); From 23009c9aafd7e1dd616773621284f33abc537616 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 12 Nov 2024 17:21:03 +0200 Subject: [PATCH 04/13] refactored defaultTbCalculatedFieldService methods --- .../entitiy/AbstractTbEntityService.java | 7 + .../entitiy/cf/CalculatedFieldCtx.java | 13 +- .../cf/DefaultTbCalculatedFieldService.java | 177 +++++++----------- .../data/cf/CalculatedFieldConfiguration.java | 3 + 4 files changed, 90 insertions(+), 110 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index d2b3890c70..cef32b9065 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -89,18 +89,25 @@ public abstract class AbstractTbEntityService { @Lazy private EntitiesVersionControlService vcService; @Autowired + @Lazy protected AccessControlService accessControlService; @Autowired + @Lazy protected TenantService tenantService; @Autowired + @Lazy protected AssetService assetService; @Autowired + @Lazy protected DeviceService deviceService; @Autowired + @Lazy protected AssetProfileService assetProfileService; @Autowired + @Lazy protected DeviceProfileService deviceProfileService; @Autowired + @Lazy protected EntityService entityService; protected boolean isTestProfile() { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java index 61f28304f2..0a7a5c8264 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java @@ -15,17 +15,20 @@ */ package org.thingsboard.server.service.entitiy.cf; -import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @Data -@Builder public class CalculatedFieldCtx { - private final CalculatedFieldId calculatedFieldId; - private final EntityId entityId; - private final CalculatedFieldState state; + private CalculatedFieldId calculatedFieldId; + private EntityId entityId; + private CalculatedFieldState state; + public CalculatedFieldCtx(CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState state) { + this.calculatedFieldId = calculatedFieldId; + this.entityId = entityId; + this.state = state; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index e222259a53..b439d8e641 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.entitiy.cf; +import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -38,7 +39,6 @@ import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -48,8 +48,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.attributes.AttributesService; @@ -62,6 +61,7 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -71,7 +71,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateEntityId; @@ -128,98 +127,6 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - private ListenableFuture> fetchAttributesForEntity(TenantId tenantId, EntityId entityId, List keys) { - return attributesService.find(tenantId, entityId, AttributeScope.SERVER_SCOPE, keys); - } - - private ListenableFuture> fetchTimeSeries(TenantId tenantId, EntityId entityId, List keys) { - return timeseriesService.findLatest(tenantId, entityId, keys); - } - - private ListenableFuture initializeStateFromFutures(TenantId tenantId, EntityId entityId, CalculatedField calculatedField, List attributeKeys, List timeSeriesKeys) { - ListenableFuture> attributesFuture = fetchAttributesForEntity(tenantId, entityId, attributeKeys); - ListenableFuture> timeSeriesFuture = fetchTimeSeries(tenantId, entityId, timeSeriesKeys); - - ListenableFuture> combinedFuture = Futures.allAsList(attributesFuture, timeSeriesFuture); - - return Futures.transform(combinedFuture, results -> { - List attributes = (List) results.get(0); - List timeSeries = (List) results.get(1); - - initializeState(calculatedField, attributes, timeSeries); - - return null; - }, MoreExecutors.directExecutor()); - } - - private void initializeState(CalculatedField calculatedField, List attributes, List timeSeries) { - CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(calculatedField.getId(), - ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); - - CalculatedFieldState state = calculatedFieldCtx.getState(); - - if (state != null) { - String calculation = performCalculation(state.getArguments()); - - Map updatedArguments = state.getArguments(); - - state = CalculatedFieldState.builder() - .arguments(updatedArguments) - .result(calculation) - .build(); - } else { - // initial calculation - Map arguments = calculatedField.getConfiguration().getArguments(); - - Map argumentValues = arguments.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> resolveArgumentValue(entry.getKey(), entry.getValue(), attributes, timeSeries) - )); - - String calculation = performCalculation(argumentValues); - - state = CalculatedFieldState.builder() - .arguments(argumentValues) - .result(calculation) - .build(); - } - - calculatedFieldCtx = new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), state); - states.put(calculatedField.getId(), calculatedFieldCtx); - } - - private String resolveArgumentValue(String key, BaseCalculatedFieldConfiguration.Argument argument, - List attributes, List timeSeries) { - String type = argument.getType(); - String value = null; - - if ("ATTRIBUTES".equals(type)) { - value = attributes.stream() - .filter(attribute -> attribute.getKey().equals(key)) - .map(AttributeKvEntry::getValueAsString) - .findFirst() - .orElse(null); - } else if ("TIME_SERIES".equals(type)) { - value = timeSeries.stream() - .filter(tsKvEntry -> tsKvEntry.getKey().equals(key)) - .map(TsKvEntry::getValueAsString) - .findFirst() - .orElse(null); - } - - return value != null ? value : argument.getDefaultValue(); - } - - private void initializeForProfile(TenantId tenantId, List links, CalculatedField cf, Iterable profileIds) { - for (T profileId : profileIds) { - for (CalculatedFieldLink link : links) { - CalculatedFieldLinkConfiguration configuration = link.getConfiguration(); - initializeStateFromFutures(tenantId, profileId, cf, configuration.getAttributes(), configuration.getTimeSeries()); - } - } - } - @Override public void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback) { try { @@ -232,21 +139,16 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); switch (entityId.getEntityType()) { - case ASSET, DEVICE -> { - for (CalculatedFieldLink link : links) { - CalculatedFieldLinkConfiguration configuration = link.getConfiguration(); - initializeStateFromFutures(tenantId, link.getEntityId(), cf, configuration.getAttributes(), configuration.getTimeSeries()); - } - } + case ASSET, DEVICE -> initializeStateForEntity(tenantId, cf, callback); case ASSET_PROFILE -> { PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - initializeForProfile(tenantId, links, cf, assetIds); + assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, callback)); } case DEVICE_PROFILE -> { PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - initializeForProfile(tenantId, links, cf, deviceIds); + deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -359,7 +261,72 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp }; } - private String performCalculation(Map argumentValues) { + private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, TbCallback callback) { + Map arguments = calculatedField.getConfiguration().getArguments(); + Map argumentValues = new HashMap<>(); + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { + @Override + public void onSuccess(Optional result) { + String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); + argumentValues.put(key, value); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to fetch data for type: {}", argument.getType(), t); + callback.onFailure(t); + } + }, calculatedFieldCallbackExecutor)); + + updateOrInitializeState(calculatedField, argumentValues); + + } + + private ListenableFuture> fetchArgumentValue(TenantId tenantId, BaseCalculatedFieldConfiguration.Argument argument) { + return switch (argument.getType()) { + case "ATTRIBUTES" -> Futures.transform( + attributesService.find(tenantId, argument.getEntityId(), AttributeScope.SERVER_SCOPE, argument.getKey()), + result -> result.map(entry -> (KvEntry) entry), + MoreExecutors.directExecutor()); + case "TIME_SERIES" -> Futures.transform( + timeseriesService.findLatest(tenantId, argument.getEntityId(), argument.getKey()), + result -> result.map(entry -> (KvEntry) entry), + MoreExecutors.directExecutor()); + default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); + }; + } + + private void updateOrInitializeState(CalculatedField calculatedField, Map argumentValues) { + CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(calculatedField.getId(), + ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); + + CalculatedFieldState state = calculatedFieldCtx.getState(); + + if (state != null) { + // calculation based on the previous data + String calculation = performCalculation(state.getArguments(), calculatedField.getConfiguration()); + + Map updatedArguments = new HashMap<>(state.getArguments()); + + state = CalculatedFieldState.builder() + .arguments(updatedArguments) + .result(calculation) + .build(); + } else { + // initial calculation + String calculation = performCalculation(argumentValues, calculatedField.getConfiguration()); + + state = CalculatedFieldState.builder() + .arguments(argumentValues) + .result(calculation) + .build(); + } + calculatedFieldCtx.setState(state); + states.put(calculatedField.getId(), calculatedFieldCtx); + } + + private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { + BaseCalculatedFieldConfiguration.Output output = calculatedFieldConfiguration.getOutput(); return "calculation"; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index deaeebcb50..cfae2e5a0e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -41,6 +41,9 @@ public interface CalculatedFieldConfiguration { Map getArguments(); + @JsonIgnore + BaseCalculatedFieldConfiguration.Output getOutput(); + @JsonIgnore List getReferencedEntities(); From 88e5da7a14068bb90bd2cb11b85217842446c220 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 13 Nov 2024 09:06:12 +0200 Subject: [PATCH 05/13] changed key for states map --- .../cf/DefaultTbCalculatedFieldService.java | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index b439d8e641..2fe1f65015 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -90,7 +90,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); - private final ConcurrentMap states = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); @Value("${state.initFetchPackSize:50000}") @Getter @@ -139,16 +139,16 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); switch (entityId.getEntityType()) { - case ASSET, DEVICE -> initializeStateForEntity(tenantId, cf, callback); + case ASSET, DEVICE -> initializeStateForEntity(tenantId, cf, entityId, callback); case ASSET_PROFILE -> { PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, callback)); + assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); } case DEVICE_PROFILE -> { PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, callback)); + deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -180,7 +180,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); calculatedFieldLinks.remove(calculatedFieldId); calculatedFields.remove(calculatedFieldId); - states.remove(calculatedFieldId); + states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); } catch (Exception e) { log.trace("Failed to process calculated field delete msg: [{}]", proto, e); callback.onFailure(e); @@ -261,7 +261,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp }; } - private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, TbCallback callback) { + private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); Map argumentValues = new HashMap<>(); arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { @@ -278,7 +278,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } }, calculatedFieldCallbackExecutor)); - updateOrInitializeState(calculatedField, argumentValues); + updateOrInitializeState(calculatedField, entityId, argumentValues); } @@ -296,8 +296,9 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp }; } - private void updateOrInitializeState(CalculatedField calculatedField, Map argumentValues) { - CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(calculatedField.getId(), + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { + String ctxId = calculatedField.getId().getId() + "_" + entityId.getId(); + CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); CalculatedFieldState state = calculatedFieldCtx.getState(); @@ -322,7 +323,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp .build(); } calculatedFieldCtx.setState(state); - states.put(calculatedField.getId(), calculatedFieldCtx); + states.put(ctxId, calculatedFieldCtx); } private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { From 4d8b62eb21e214cbbf2940a4ded8bac678eadf3c Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 13 Nov 2024 09:55:03 +0200 Subject: [PATCH 06/13] moved logic of cf add/update/delete msg to a single method --- .../cf/DefaultTbCalculatedFieldService.java | 55 +++++++++---------- .../entitiy/cf/TbCalculatedFieldService.java | 6 +- common/proto/src/main/proto/queue.proto | 19 ++----- 3 files changed, 30 insertions(+), 50 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 2fe1f65015..4c83fcf7b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -128,10 +128,17 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } @Override - public void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback) { + public void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + if (proto.getDeleted()) { + onCalculatedFieldDelete(calculatedFieldId, callback); + callback.onSuccess(); + } + if (proto.getUpdated()) { + onCalculatedFieldDelete(calculatedFieldId, callback); + } CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cf != null) { @@ -163,30 +170,6 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - @Override - public void onCalculatedFieldUpdated(TransportProtos.CalculatedFieldUpdateMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); - } catch (Exception e) { - log.trace("Failed to process calculated field update msg: [{}]", proto, e); - callback.onFailure(e); - } - } - - @Override - public void onCalculatedFieldDeleted(TransportProtos.CalculatedFieldDeleteMsgProto proto, TbCallback callback) { - try { - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); - calculatedFieldLinks.remove(calculatedFieldId); - calculatedFields.remove(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); - } catch (Exception e) { - log.trace("Failed to process calculated field delete msg: [{}]", proto, e); - callback.onFailure(e); - } - } - @Override public CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException { ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; @@ -223,13 +206,25 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } + + private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { + try { + calculatedFieldLinks.remove(calculatedFieldId); + calculatedFields.remove(calculatedFieldId); + states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); + } catch (Exception e) { + log.trace("Failed to delete calculated field.", e); + callback.onFailure(e); + } + } + private void fetchCalculatedFields() { PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); // TODO: read all states(CalculatedFieldCtx) - states.keySet().removeIf(calculatedFieldId -> !calculatedFields.containsKey(calculatedFieldId)); + states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.startsWith(id.toString()))); } private void checkEntityExistence(TenantId tenantId, EntityId entityId) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java index 4f14270c3d..aa77d29702 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -24,11 +24,7 @@ import org.thingsboard.server.service.security.model.SecurityUser; public interface TbCalculatedFieldService { - void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback); - - void onCalculatedFieldUpdated(TransportProtos.CalculatedFieldUpdateMsgProto proto, TbCallback callback); - - void onCalculatedFieldDeleted(TransportProtos.CalculatedFieldDeleteMsgProto proto, TbCallback callback); + void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index e1d6320779..b41e010a2d 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1267,25 +1267,14 @@ message ToDeviceActorNotificationMsgProto { DeviceDeleteMsgProto deviceDeleteMsg = 8; } -message CalculatedFieldAddMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 calculatedFieldIdMSB = 3; - int64 calculatedFieldIdLSB = 4; -} - -message CalculatedFieldUpdateMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 calculatedFieldIdMSB = 3; - int64 calculatedFieldIdLSB = 4; -} - -message CalculatedFieldDeleteMsgProto { +message CalculatedFieldMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; int64 calculatedFieldIdMSB = 3; int64 calculatedFieldIdLSB = 4; + bool added = 5; + bool updated = 6; + bool deleted = 7; } /** From 3072861a8fa986cda6b57561a343fd3d1c6c094b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 13 Nov 2024 17:42:55 +0200 Subject: [PATCH 07/13] implemented methods for calculated field update/delete in cluster service --- application/pom.xml | 4 + .../entitiy/EntityStateSourcingListener.java | 28 +++++- .../cf/DefaultTbCalculatedFieldService.java | 11 ++- .../service/entitiy/cf/RocksDBService.java | 95 +++++++++++++++++++ .../queue/DefaultTbClusterService.java | 34 ++++++- .../queue/DefaultTbCoreConsumerService.java | 21 +++- .../server/utils/RocksDBConfig.java | 52 ++++++++++ .../src/main/resources/thingsboard.yml | 4 + .../server/cluster/TbClusterService.java | 5 + common/proto/src/main/proto/queue.proto | 1 + .../dao/cf/BaseCalculatedFieldService.java | 7 +- pom.xml | 7 ++ 12 files changed, 260 insertions(+), 9 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java create mode 100644 application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java diff --git a/application/pom.xml b/application/pom.xml index 17b179c767..62d2a49908 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -369,6 +369,10 @@ com.google.firebase firebase-admin + + org.rocksdb + rocksdbjni + diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 863be23e42..6c2e995b82 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.DeviceId; @@ -118,7 +119,11 @@ public class EntityStateSourcingListener { ApiUsageState apiUsageState = (ApiUsageState) event.getEntity(); tbClusterService.onApiStateChange(apiUsageState, null); } - default -> {} + case CALCULATED_FIELD -> { + onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity()); + } + default -> { + } } } @@ -130,7 +135,7 @@ public class EntityStateSourcingListener { return; } EntityType entityType = entityId.getEntityType(); - if (!tenantId.isSysTenantId() && entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { + if (!tenantId.isSysTenantId() && entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { log.debug("[{}] Ignoring DeleteEntityEvent because tenant does not exist: {}", tenantId, event); return; } @@ -149,7 +154,8 @@ public class EntityStateSourcingListener { case RULE_CHAIN -> { RuleChain ruleChain = (RuleChain) event.getEntity(); if (RuleChainType.CORE.equals(ruleChain.getType())) { - Set referencingRuleChainIds = JacksonUtil.fromString(event.getBody(), new TypeReference<>() {}); + Set referencingRuleChainIds = JacksonUtil.fromString(event.getBody(), new TypeReference<>() { + }); if (referencingRuleChainIds != null) { referencingRuleChainIds.forEach(referencingRuleChainId -> tbClusterService.broadcastEntityStateChangeEvent(tenantId, referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); @@ -177,7 +183,12 @@ public class EntityStateSourcingListener { TbResourceInfo tbResource = (TbResourceInfo) event.getEntity(); tbClusterService.onResourceDeleted(tbResource, null); } - default -> {} + case CALCULATED_FIELD -> { + CalculatedField calculatedField = (CalculatedField) event.getEntity(); + tbClusterService.onCalculatedFieldDeleted(tenantId, calculatedField, null); + } + default -> { + } } } @@ -247,6 +258,15 @@ public class EntityStateSourcingListener { } } + private void onCalculatedFieldUpdate(Object entity, Object oldEntity) { + CalculatedField calculatedField = (CalculatedField) entity; + CalculatedField oldCalculatedField = null; + if (oldEntity instanceof CalculatedField) { + oldCalculatedField = (CalculatedField) oldEntity; + } + tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField); + } + private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { String data = JacksonUtil.toString(JacksonUtil.valueToTree(assignedDevice)); if (data != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 4c83fcf7b5..06c87ff4d1 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -29,6 +29,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.AttributeScope; @@ -64,6 +65,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Random; import java.util.UUID; @@ -71,6 +73,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateEntityId; @@ -83,6 +86,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private final CalculatedFieldService calculatedFieldService; private final AttributesService attributesService; private final TimeseriesService timeseriesService; + private final RocksDBService rocksDBService; private ListeningScheduledExecutorService scheduledExecutor; private ListeningExecutorService calculatedFieldExecutor; @@ -212,6 +216,10 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFieldLinks.remove(calculatedFieldId); calculatedFields.remove(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); + List statesToRemove = states.keySet().stream() + .filter(key -> key.startsWith(calculatedFieldId.getId().toString())) + .collect(Collectors.toList()); + rocksDBService.deleteAll(statesToRemove); } catch (Exception e) { log.trace("Failed to delete calculated field.", e); callback.onFailure(e); @@ -223,7 +231,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - // TODO: read all states(CalculatedFieldCtx) + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(ctxId, JacksonUtil.convertValue(ctx, CalculatedFieldCtx.class))); states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.startsWith(id.toString()))); } @@ -319,6 +327,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } calculatedFieldCtx.setState(state); states.put(ctxId, calculatedFieldCtx); + rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.toString(calculatedFieldCtx))); } private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java new file mode 100644 index 0000000000..2ba3e9bb4f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java @@ -0,0 +1,95 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.extern.slf4j.Slf4j; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteBatch; +import org.rocksdb.WriteOptions; +import org.springframework.stereotype.Service; +import org.thingsboard.server.utils.RocksDBConfig; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service +@Slf4j +public class RocksDBService { + + private final RocksDB db; + private final WriteOptions writeOptions; + + public RocksDBService(RocksDBConfig config) throws RocksDBException { + this.db = config.getDb(); + this.writeOptions = new WriteOptions().setSync(true); + } + + public void put(String key, String value) { + try { + db.put(writeOptions, key.getBytes(StandardCharsets.UTF_8), value.getBytes(StandardCharsets.UTF_8)); + } catch (RocksDBException e) { + log.error("Failed to store data to RocksDB", e); + } + } + + public void delete(String key) { + try { + db.delete(writeOptions, key.getBytes(StandardCharsets.UTF_8)); + } catch (RocksDBException e) { + log.error("Failed to delete data from RocksDB", e); + } + } + + public void deleteAll(List keys) { + try (WriteBatch batch = new WriteBatch()) { + for (String key : keys) { + batch.delete(key.getBytes(StandardCharsets.UTF_8)); + } + db.write(writeOptions, batch); + } catch (RocksDBException e) { + log.error("Failed to delete data from RocksDB", e); + } + } + + public String get(String key) { + try { + byte[] value = db.get(key.getBytes(StandardCharsets.UTF_8)); + return value != null ? new String(value, StandardCharsets.UTF_8) : null; + } catch (RocksDBException e) { + log.error("Failed to retrieve data from RocksDB", e); + return null; + } + } + + public Map getAll() { + Map map = new HashMap<>(); + try (RocksIterator iterator = db.newIterator()) { + for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { + String key = new String(iterator.key(), StandardCharsets.UTF_8); + String value = new String(iterator.value(), StandardCharsets.UTF_8); + map.put(key, value); + } + } catch (Exception e) { + log.error("Failed to retrieve data from RocksDB", e); + } + return map; + } + +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 301f8e8838..f7c8230716 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -38,10 +38,12 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EdgeId; @@ -666,7 +668,8 @@ public class DefaultTbClusterService implements TbClusterService { private void pushDeviceUpdateMessage(TenantId tenantId, EdgeId edgeId, EntityId entityId, EdgeEventActionType action) { log.trace("{} Going to send edge update notification for device actor, device id {}, edge id {}", tenantId, entityId, edgeId); switch (action) { - case ASSIGNED_TO_EDGE -> pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); + case ASSIGNED_TO_EDGE -> + pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); case UNASSIGNED_FROM_EDGE -> { EdgeId relatedEdgeId = findRelatedEdgeIdIfAny(tenantId, entityId); pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), relatedEdgeId), null); @@ -743,4 +746,33 @@ public class DefaultTbClusterService implements TbClusterService { } } + @Override + public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField) { + var created = oldCalculatedField == null; + broadcastEntityChangeToTransport(calculatedField.getTenantId(), calculatedField.getId(), calculatedField, null); + broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + sendCalculatedFieldEvent(calculatedField.getTenantId(), calculatedField.getId(), created, !created, false); + } + + @Override + public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback) { + CalculatedFieldId calculatedFieldId = calculatedField.getId(); + broadcastEntityDeleteToTransport(tenantId, calculatedFieldId, calculatedField.getName(), callback); + sendCalculatedFieldEvent(tenantId, calculatedFieldId, false, false, true); + broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); + } + + private void sendCalculatedFieldEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, boolean added, boolean updated, boolean deleted) { + TransportProtos.CalculatedFieldMsgProto.Builder builder = TransportProtos.CalculatedFieldMsgProto.newBuilder(); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + builder.setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()); + builder.setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()); + builder.setAdded(added); + builder.setUpdated(updated); + builder.setDeleted(deleted); + TransportProtos.CalculatedFieldMsgProto msg = builder.build(); + pushMsgToCore(tenantId, calculatedFieldId, ToCoreMsg.newBuilder().setCalculatedFieldMsg(msg).build(), null); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 2a64dd3388..979c419003 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.NotificationRequestId; import org.thingsboard.server.common.data.id.TenantId; @@ -85,6 +86,7 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -148,6 +150,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig> mainConsumer; private QueueConsumerManager> usageStatsConsumer; @@ -175,7 +178,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = deviceActivityEventsExecutor.submit(() -> calculatedFieldService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); + callback.onFailure(t); + }); + } + private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java b/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java new file mode 100644 index 0000000000..75a4dd138a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils; + +import jakarta.annotation.PreDestroy; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class RocksDBConfig { + + @Value("${rocksdb.db_path}") + private String dbPath; + private RocksDB db; + + static { + RocksDB.loadLibrary(); + } + + public RocksDB getDb() throws RocksDBException { + if (db == null) { + Options options = new Options().setCreateIfMissing(true); + db = RocksDB.open(options, dbPath); + } + return db; + } + + @PreDestroy + public void close() { + if (db != null) { + db.close(); + db = null; + } + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 106810a7a1..8b1aeedec8 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -424,6 +424,10 @@ sql: pool_size: "${SQL_RELATIONS_POOL_SIZE:4}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls query_timeout: "${SQL_RELATIONS_QUERY_TIMEOUT_SEC:20}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls +rocksdb: + # Rocksdb path + db_path: "${ROCKS_DB_PATH:}" + # Actor system parameters actors: system: diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 5112e93da7..f173005107 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.EdgeId; @@ -114,4 +115,8 @@ public interface TbClusterService extends TbQueueClusterService { void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId sourceEdgeId); + void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField); + + void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback); + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index b41e010a2d..95e4fa8601 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1512,6 +1512,7 @@ message ToCoreMsg { DeviceConnectProto deviceConnectMsg = 50; DeviceDisconnectProto deviceDisconnectMsg = 51; DeviceInactivityProto deviceInactivityMsg = 52; + CalculatedFieldMsgProto calculatedFieldMsg = 53; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index db5539eb96..8603dadcc9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -29,20 +29,21 @@ import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; -import static org.thingsboard.server.dao.entity.AbstractEntityService.checkConstraintViolation; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validatePageLink; @Service("CalculatedFieldDaoService") @Slf4j @RequiredArgsConstructor -public class BaseCalculatedFieldService implements CalculatedFieldService { +public class BaseCalculatedFieldService extends AbstractEntityService implements CalculatedFieldService { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; public static final String INCORRECT_CALCULATED_FIELD_ID = "Incorrect calculatedFieldId "; @@ -60,6 +61,8 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { log.trace("Executing save calculated field, [{}]", calculatedField); CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); + eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedCalculatedField.getTenantId()).entityId(savedCalculatedField.getId()) + .entity(savedCalculatedField).created(calculatedField.getId() == null).build()); return savedCalculatedField; } catch (Exception e) { checkConstraintViolation(e, diff --git a/pom.xml b/pom.xml index a1d5f26ecf..9b835516fc 100755 --- a/pom.xml +++ b/pom.xml @@ -166,6 +166,8 @@ 1.6.1 2.19.0 9.2.0 + + 9.4.0 @@ -2272,6 +2274,11 @@ metadata-extractor ${drewnoakes-metadata-extractor.version} + + org.rocksdb + rocksdbjni + ${rocksdbjni.version} + From f75eaf8226cff33ad0d8f935f1750394a3475375 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 14 Nov 2024 10:23:57 +0200 Subject: [PATCH 08/13] added rocksdb path --- .../entitiy/AbstractTbEntityService.java | 7 ------- .../server/utils/RocksDBConfig.java | 2 +- .../src/main/resources/thingsboard.yml | 2 +- .../alarm/DefaultTbAlarmServiceTest.java | 21 +++++++++++++++++++ .../DefaultTbAlarmCommentServiceTest.java | 21 +++++++++++++++++++ 5 files changed, 44 insertions(+), 9 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index cef32b9065..d2b3890c70 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -89,25 +89,18 @@ public abstract class AbstractTbEntityService { @Lazy private EntitiesVersionControlService vcService; @Autowired - @Lazy protected AccessControlService accessControlService; @Autowired - @Lazy protected TenantService tenantService; @Autowired - @Lazy protected AssetService assetService; @Autowired - @Lazy protected DeviceService deviceService; @Autowired - @Lazy protected AssetProfileService assetProfileService; @Autowired - @Lazy protected DeviceProfileService deviceProfileService; @Autowired - @Lazy protected EntityService entityService; protected boolean isTestProfile() { diff --git a/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java b/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java index 75a4dd138a..9c3c02f472 100644 --- a/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java +++ b/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java @@ -25,7 +25,7 @@ import org.springframework.stereotype.Component; @Component public class RocksDBConfig { - @Value("${rocksdb.db_path}") + @Value("${rocksdb.db_path:${java.io.tmpdir}/rocksdb}") private String dbPath; private RocksDB db; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 8b1aeedec8..97405f35d3 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -426,7 +426,7 @@ sql: rocksdb: # Rocksdb path - db_path: "${ROCKS_DB_PATH:}" + db_path: "${ROCKS_DB_PATH:${java.io.tmpdir}/rocksdb}" # Actor system parameters actors: diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java index 60a0ac3bf6..bc1726ff41 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java @@ -38,10 +38,17 @@ import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.entitiy.TbLogEntityActionService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; @@ -81,6 +88,20 @@ public class DefaultTbAlarmServiceTest { protected TbClusterService tbClusterService; @MockBean private EntitiesVersionControlService vcService; + @MockBean + private AccessControlService accessControlService; + @MockBean + private TenantService tenantService; + @MockBean + private AssetService assetService; + @MockBean + private DeviceService deviceService; + @MockBean + private AssetProfileService assetProfileService; + @MockBean + private DeviceProfileService deviceProfileService; + @MockBean + private EntityService entityService; @SpyBean DefaultTbAlarmService service; diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java index b36ca51416..71c295edc9 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java @@ -35,10 +35,17 @@ import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.dao.alarm.AlarmCommentService; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.entitiy.TbLogEntityActionService; import org.thingsboard.server.service.entitiy.alarm.DefaultTbAlarmCommentService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import java.util.UUID; @@ -72,6 +79,20 @@ public class DefaultTbAlarmCommentServiceTest { protected CustomerService customerService; @MockBean protected TbClusterService tbClusterService; + @MockBean + private AccessControlService accessControlService; + @MockBean + private TenantService tenantService; + @MockBean + private AssetService assetService; + @MockBean + private DeviceService deviceService; + @MockBean + private AssetProfileService assetProfileService; + @MockBean + private DeviceProfileService deviceProfileService; + @MockBean + private EntityService entityService; @SpyBean DefaultTbAlarmCommentService service; From 8079ef66efc3be7f54305758390c296cd6848356 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 14 Nov 2024 14:44:40 +0200 Subject: [PATCH 09/13] added logs --- .../entitiy/cf/CalculatedFieldCtx.java | 3 ++ .../entitiy/cf/CalculatedFieldState.java | 1 + .../cf/DefaultTbCalculatedFieldService.java | 29 ++++++++++++------- .../data/cf/CalculatedFieldConfiguration.java | 1 - .../dao/model/sql/CalculatedFieldEntity.java | 8 ++--- .../model/sql/CalculatedFieldLinkEntity.java | 4 +-- ...efaultNativeCalculatedFieldRepository.java | 4 +-- 7 files changed, 31 insertions(+), 19 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java index 0a7a5c8264..021a9bbc83 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java @@ -26,6 +26,9 @@ public class CalculatedFieldCtx { private EntityId entityId; private CalculatedFieldState state; + public CalculatedFieldCtx() { + } + public CalculatedFieldCtx(CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState state) { this.calculatedFieldId = calculatedFieldId; this.entityId = entityId; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java index dc07b820ea..f84f4a57c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -24,6 +24,7 @@ import java.util.Map; @Builder public class CalculatedFieldState { + // TODO: use value object(TsKv) instead of string Map arguments; String result; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 06c87ff4d1..51ff540a64 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -67,7 +67,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Random; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -113,7 +112,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.scheduleWithFixedDelay(this::fetchCalculatedFields, new Random().nextInt(defaultCalculatedFieldCheckIntervalInSec), defaultCalculatedFieldCheckIntervalInSec, TimeUnit.SECONDS); + scheduledExecutor.scheduleWithFixedDelay(this::fetchCalculatedFields, 0, defaultCalculatedFieldCheckIntervalInSec, TimeUnit.SECONDS); } @PreDestroy @@ -136,11 +135,15 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); if (proto.getDeleted()) { + log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } if (proto.getUpdated()) { + log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); + //TODO: improve the check. Maybe it was renamed or just the name of the result changed. onCalculatedFieldDelete(calculatedFieldId, callback); } CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); @@ -150,13 +153,18 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); switch (entityId.getEntityType()) { - case ASSET, DEVICE -> initializeStateForEntity(tenantId, cf, entityId, callback); + case ASSET, DEVICE -> { + log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); + initializeStateForEntity(tenantId, cf, entityId, callback); + } case ASSET_PROFILE -> { + log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); } case DEVICE_PROFILE -> { + log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); @@ -164,12 +172,14 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); } + log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); } else { - //Calculated field or entity was probably deleted while message was in queue; + //Calculated field was probably deleted while message was in queue; + log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); callback.onSuccess(); } } catch (Exception e) { - log.trace("Failed to process calculated field add msg: [{}]", proto, e); + log.trace("Failed to process calculated field msg: [{}]", proto, e); callback.onFailure(e); } } @@ -210,7 +220,6 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { try { calculatedFieldLinks.remove(calculatedFieldId); @@ -221,7 +230,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp .collect(Collectors.toList()); rocksDBService.deleteAll(statesToRemove); } catch (Exception e) { - log.trace("Failed to delete calculated field.", e); + log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); callback.onFailure(e); } } @@ -231,7 +240,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(ctxId, JacksonUtil.convertValue(ctx, CalculatedFieldCtx.class))); + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(ctxId, JacksonUtil.fromString(ctx, CalculatedFieldCtx.class))); states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.startsWith(id.toString()))); } @@ -276,7 +285,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp @Override public void onFailure(Throwable t) { - log.warn("Failed to fetch data for type: {}", argument.getType(), t); + log.warn("Failed to initialize state for entity: [{}]", entityId, t); callback.onFailure(t); } }, calculatedFieldCallbackExecutor)); @@ -327,7 +336,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } calculatedFieldCtx.setState(state); states.put(ctxId, calculatedFieldCtx); - rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.toString(calculatedFieldCtx))); + rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.writeValueAsString(calculatedFieldCtx))); } private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index cfae2e5a0e..7e6a77f006 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -41,7 +41,6 @@ public interface CalculatedFieldConfiguration { Map getArguments(); - @JsonIgnore BaseCalculatedFieldConfiguration.Output getOutput(); @JsonIgnore diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 725ddd7bef..efb262477b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -56,7 +56,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem private UUID tenantId; @Column(name = CALCULATED_FIELD_ENTITY_TYPE) - private EntityType entityType; + private String entityType; @Column(name = CALCULATED_FIELD_ENTITY_ID) private UUID entityId; @@ -88,12 +88,12 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.setUuid(calculatedField.getUuidId()); this.createdTime = calculatedField.getCreatedTime(); this.tenantId = calculatedField.getTenantId().getId(); - this.entityType = calculatedField.getEntityId().getEntityType(); + this.entityType = calculatedField.getEntityId().getEntityType().name(); this.entityId = calculatedField.getEntityId().getId(); this.type = calculatedField.getType(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); - this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(entityType, entityId); + this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(EntityType.valueOf(entityType), entityId); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { this.externalId = calculatedField.getExternalId().getId(); @@ -109,7 +109,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setType(type); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, entityType, entityId)); + calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, EntityType.valueOf(entityType), entityId)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 0ac5a9bc94..b4d6677be4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -53,7 +53,7 @@ public class CalculatedFieldLinkEntity extends BaseSqlEntity Date: Thu, 14 Nov 2024 17:19:14 +0200 Subject: [PATCH 10/13] added logic to handle cf update msg --- .../entitiy/cf/CalculatedFieldState.java | 24 ++++--- .../cf/DefaultTbCalculatedFieldService.java | 67 ++++++++++++------- .../cf/SimpleCalculatedFieldState.java | 49 ++++++++++++++ .../CalculatedFieldControllerTest.java | 3 +- .../cf/BaseCalculatedFieldConfiguration.java | 1 + .../common/data/cf/CalculatedField.java | 12 ++-- .../data/cf/CalculatedFieldConfiguration.java | 2 +- .../common/data/cf/CalculatedFieldType.java | 22 ++++++ .../SimpleCalculatedFieldConfiguration.java | 4 +- .../dao/model/sql/CalculatedFieldEntity.java | 9 +-- ...efaultNativeCalculatedFieldRepository.java | 7 +- .../server/dao/service/AssetServiceTest.java | 3 +- .../service/CalculatedFieldServiceTest.java | 3 +- .../dao/service/CustomerServiceTest.java | 3 +- .../server/dao/service/DeviceServiceTest.java | 3 +- 15 files changed, 160 insertions(+), 52 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java index f84f4a57c2..9fb6a009fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -15,17 +15,25 @@ */ package org.thingsboard.server.service.entitiy.cf; -import lombok.Builder; -import lombok.Data; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import java.util.Map; -@Data -@Builder -public class CalculatedFieldState { +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE") +}) +public interface CalculatedFieldState { - // TODO: use value object(TsKv) instead of string - Map arguments; - String result; + CalculatedFieldType getType(); + + void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 51ff540a64..49c64a1262 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -141,12 +142,11 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } + CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); - //TODO: improve the check. Maybe it was renamed or just the name of the result changed. - onCalculatedFieldDelete(calculatedFieldId, callback); + onCalculatedFieldUpdate(cf, callback); } - CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cf != null) { EntityId entityId = cf.getEntityId(); @@ -235,6 +235,31 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } + private void onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { + CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); + if (hasSignificantChanged(oldCalculatedField, newCalculatedField)) { + onCalculatedFieldDelete(newCalculatedField.getId(), callback); + } else { + calculatedFields.put(newCalculatedField.getId(), newCalculatedField); + callback.onSuccess(); + } + } + + private boolean hasSignificantChanged(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + if (oldCalculatedField == null) { + return true; + } + boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); + boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); + CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); + CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); + boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); + boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); + boolean outputNameChanged = !oldConfig.getOutput().getName().equals(newConfig.getOutput().getName()); + + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputNameChanged; + } + private void fetchCalculatedFields() { PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); @@ -309,39 +334,33 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { - String ctxId = calculatedField.getId().getId() + "_" + entityId.getId(); + String ctxId = generateCtxId(calculatedField.getId(), entityId); CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); CalculatedFieldState state = calculatedFieldCtx.getState(); - if (state != null) { - // calculation based on the previous data - String calculation = performCalculation(state.getArguments(), calculatedField.getConfiguration()); - - Map updatedArguments = new HashMap<>(state.getArguments()); - - state = CalculatedFieldState.builder() - .arguments(updatedArguments) - .result(calculation) - .build(); + state.performCalculation(argumentValues, calculatedField.getConfiguration(), false); } else { - // initial calculation - String calculation = performCalculation(argumentValues, calculatedField.getConfiguration()); - - state = CalculatedFieldState.builder() - .arguments(argumentValues) - .result(calculation) - .build(); + CalculatedFieldState newState = createStateByType(calculatedField.getType()); + newState.performCalculation(argumentValues, calculatedField.getConfiguration(), true); } calculatedFieldCtx.setState(state); + states.put(ctxId, calculatedFieldCtx); rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.writeValueAsString(calculatedFieldCtx))); } - private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { - BaseCalculatedFieldConfiguration.Output output = calculatedFieldConfiguration.getOutput(); - return "calculation"; + private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { + return switch (calculatedFieldType) { + case SIMPLE -> new SimpleCalculatedFieldState(); + default -> + throw new IllegalArgumentException("Invalid calculated field type '" + calculatedFieldType + "'."); + }; + } + + private String generateCtxId(CalculatedFieldId calculatedFieldId, EntityId entityId) { + return calculatedFieldId.getId() + "_" + entityId.getId(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java new file mode 100644 index 0000000000..8a90893262 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +import java.util.HashMap; +import java.util.Map; + +@Data +public class SimpleCalculatedFieldState implements CalculatedFieldState { + + // TODO: use value object(TsKv) instead of string + Map arguments = new HashMap<>(); + String result; + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SIMPLE; + } + + @Override + public void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation) { + if (initialCalculation) { + // todo: perform initial calculation + this.arguments = argumentValues; + } else { + // todo: perform calculation based on previous data + this.arguments.putAll(argumentValues); + } + this.result = "result"; + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 67e7bc64bd..ebda2cbf82 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -124,7 +125,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { private CalculatedField getCalculatedField(DeviceId deviceId) { CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(deviceId); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); calculatedField.setConfiguration(getCalculatedFieldConfig(null)); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java index 3894835abc..4575e414ac 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java @@ -115,6 +115,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel @Data public static class Output { + private String name; private String type; private String expression; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index d96de37a39..ceb1222fe2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -20,15 +20,17 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; -import org.thingsboard.server.common.data.*; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.HasVersion; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; -import java.io.Serializable; - @Schema @Data @EqualsAndHashCode(callSuper = true) @@ -41,7 +43,7 @@ public class CalculatedField extends BaseData implements HasN @NoXss @Length(fieldName = "type") - private String type; + private CalculatedFieldType type; @NoXss @Length(fieldName = "name") @Schema(description = "User defined name of the calculated field.") @@ -65,7 +67,7 @@ public class CalculatedField extends BaseData implements HasN super(id); } - public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version, CalculatedFieldId externalId) { + public CalculatedField(TenantId tenantId, EntityId entityId, CalculatedFieldType type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version, CalculatedFieldId externalId) { this.tenantId = tenantId; this.entityId = entityId; this.type = type; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index 7e6a77f006..f733c35310 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -37,7 +37,7 @@ import java.util.UUID; public interface CalculatedFieldConfiguration { @JsonIgnore - String getType(); + CalculatedFieldType getType(); Map getArguments(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java new file mode 100644 index 0000000000..89173b35b9 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +public enum CalculatedFieldType { + + SIMPLE, SCRIPT + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java index d635e0d82e..327f9cdc75 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java @@ -33,7 +33,7 @@ public class SimpleCalculatedFieldConfiguration extends BaseCalculatedFieldConfi } @Override - public String getType() { - return "SIMPLE"; + public CalculatedFieldType getType() { + return CalculatedFieldType.SIMPLE; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index efb262477b..3c45a81cb5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -25,6 +25,7 @@ import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -90,7 +91,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.tenantId = calculatedField.getTenantId().getId(); this.entityType = calculatedField.getEntityId().getEntityType().name(); this.entityId = calculatedField.getEntityId().getId(); - this.type = calculatedField.getType(); + this.type = calculatedField.getType().name(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(EntityType.valueOf(entityType), entityId); @@ -106,7 +107,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setCreatedTime(createdTime); calculatedField.setTenantId(TenantId.fromUUID(tenantId)); calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); - calculatedField.setType(type); + calculatedField.setType(CalculatedFieldType.valueOf(type)); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, EntityType.valueOf(entityType), entityId)); @@ -118,8 +119,8 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem } private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - switch (type) { - case "SIMPLE": + switch (CalculatedFieldType.valueOf(type)) { + case SIMPLE: return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); default: throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index 73f334e8f1..417a468b2c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; @@ -73,7 +74,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF UUID tenantId = (UUID) row.get("tenant_id"); EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); UUID entityId = (UUID) row.get("entity_id"); - String type = (String) row.get("type"); + CalculatedFieldType type = CalculatedFieldType.valueOf((String) row.get("type")); String name = (String) row.get("name"); int configurationVersion = (int) row.get("configuration_version"); JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); @@ -133,9 +134,9 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF }); } - private CalculatedFieldConfiguration readCalculatedFieldConfiguration(String type, JsonNode config, EntityType entityType, UUID entityId) { + private CalculatedFieldConfiguration readCalculatedFieldConfiguration(CalculatedFieldType type, JsonNode config, EntityType entityType, UUID entityId) { switch (type) { - case "SIMPLE": + case SIMPLE: return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); default: throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 432d559598..43b51ce2ac 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -874,7 +875,7 @@ public class AssetServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setEntityId(savedAssetWithCf.getId()); SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index a94eb83fa3..d5d025a46d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -25,6 +25,7 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -137,7 +138,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setEntityId(entityId); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 0e2d9b797b..1c5e0d8f49 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -369,7 +370,7 @@ public class CustomerServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setEntityId(savedAsset.getId()); SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 33b80c0d99..8f1f16e631 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -1212,7 +1213,7 @@ public class DeviceServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setEntityId(deviceWithCf.getId()); SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); From 7c433d47e2db2b291ebd99d2047f3c2e10887178 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 18 Nov 2024 08:46:35 +0200 Subject: [PATCH 11/13] changed type value in cf data validator test --- .../service/validator/CalculatedFieldDataValidatorTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java index 6d0eb0ad38..10df39e04b 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java @@ -20,6 +20,7 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.SpyBean; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.cf.CalculatedFieldDao; @@ -44,7 +45,7 @@ public class CalculatedFieldDataValidatorTest { @Test public void testUpdateNonExistingCalculatedField() { CalculatedField calculatedField = new CalculatedField(CALCULATED_FIELD_ID); - calculatedField.setType("Simple"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setName("Test"); given(calculatedFieldDao.findById(TENANT_ID, CALCULATED_FIELD_ID.getId())).willReturn(null); From 2fc32ee232c37cb9201e18adbebcc4d5e5ab7b24 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 18 Nov 2024 12:07:41 +0200 Subject: [PATCH 12/13] removed unnecessary methods --- .../entitiy/cf/CalculatedFieldState.java | 2 + .../cf/DefaultTbCalculatedFieldService.java | 24 +++++--- .../src/main/resources/thingsboard.yml | 4 ++ .../server/dao/cf/CalculatedFieldService.java | 9 ++- .../dao/cf/BaseCalculatedFieldService.java | 55 ++++++++++++++++--- .../server/dao/cf/CalculatedFieldDao.java | 2 - .../server/dao/cf/CalculatedFieldLinkDao.java | 3 + .../dao/sql/cf/JpaCalculatedFieldDao.java | 7 --- .../dao/sql/cf/JpaCalculatedFieldLinkDao.java | 7 ++- 9 files changed, 84 insertions(+), 29 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java index 9fb6a009fe..221c44b94c 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.entitiy.cf; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; @@ -32,6 +33,7 @@ import java.util.Map; }) public interface CalculatedFieldState { + @JsonIgnore CalculatedFieldType getType(); void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 49c64a1262..f2f16de844 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -96,7 +96,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); - @Value("${state.initFetchPackSize:50000}") + @Value("${calculatedField.initFetchPackSize:50000}") @Getter private int initFetchPackSize; @@ -145,7 +145,10 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); - onCalculatedFieldUpdate(cf, callback); + boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); + if (!shouldReinit) { + return; + } } List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cf != null) { @@ -172,12 +175,13 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); } - log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); } else { //Calculated field was probably deleted while message was in queue; log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); callback.onSuccess(); } + callback.onSuccess(); + log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to process calculated field msg: [{}]", proto, e); callback.onFailure(e); @@ -235,17 +239,20 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - private void onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { + private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); - if (hasSignificantChanged(oldCalculatedField, newCalculatedField)) { + boolean shouldReinit = true; + if (hasSignificantChanges(oldCalculatedField, newCalculatedField)) { onCalculatedFieldDelete(newCalculatedField.getId(), callback); } else { calculatedFields.put(newCalculatedField.getId(), newCalculatedField); callback.onSuccess(); + shouldReinit = false; } + return shouldReinit; } - private boolean hasSignificantChanged(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { if (oldCalculatedField == null) { return true; } @@ -255,9 +262,9 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); - boolean outputNameChanged = !oldConfig.getOutput().getName().equals(newConfig.getOutput().getName()); + boolean outputExpressionChanged = !oldConfig.getOutput().getExpression().equals(newConfig.getOutput().getExpression()); - return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputNameChanged; + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputExpressionChanged; } private void fetchCalculatedFields() { @@ -344,6 +351,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } else { CalculatedFieldState newState = createStateByType(calculatedField.getType()); newState.performCalculation(argumentValues, calculatedField.getConfiguration(), true); + state = newState; } calculatedFieldCtx.setState(state); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 97405f35d3..699d2012cc 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -424,6 +424,10 @@ sql: pool_size: "${SQL_RELATIONS_POOL_SIZE:4}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls query_timeout: "${SQL_RELATIONS_QUERY_TIMEOUT_SEC:20}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls +# Calculated Field parameters +calculatedField: + initFetchPackSize: "${INIT_FETCH_PACK_SIZE:50000}" + rocksdb: # Rocksdb path db_path: "${ROCKS_DB_PATH:${java.io.tmpdir}/rocksdb}" diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index c12acade6c..4bb67f9f0d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.cf; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -33,6 +34,8 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + ListenableFuture findCalculatedFieldByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findAllCalculatedFields(); PageData findAllCalculatedFields(PageLink pageLink); @@ -45,13 +48,15 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedFieldLink findCalculatedFieldLinkById(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); + ListenableFuture findCalculatedFieldLinkByIdAsync(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); + List findAllCalculatedFieldLinks(); List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId); - PageData findAllCalculatedFieldLinks(PageLink pageLink); + ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); - boolean existsByEntityId(TenantId tenantId, EntityId entityId); + PageData findAllCalculatedFieldLinks(PageLink pageLink); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 8603dadcc9..fd669f7865 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.cf; +import com.google.common.util.concurrent.ListenableFuture; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -30,7 +31,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; +import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; @@ -55,14 +58,14 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements @Override public CalculatedField save(CalculatedField calculatedField) { - calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); + CalculatedField oldCalculatedField = calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); try { TenantId tenantId = calculatedField.getTenantId(); log.trace("Executing save calculated field, [{}]", calculatedField); CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedCalculatedField.getTenantId()).entityId(savedCalculatedField.getId()) - .entity(savedCalculatedField).created(calculatedField.getId() == null).build()); + .entity(savedCalculatedField).oldEntity(oldCalculatedField).created(calculatedField.getId() == null).build()); return savedCalculatedField; } catch (Exception e) { checkConstraintViolation(e, @@ -80,6 +83,13 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldDao.findById(tenantId, calculatedFieldId.getId()); } + @Override + public ListenableFuture findCalculatedFieldByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findCalculatedFieldByIdAsync [{}]", calculatedFieldId); + validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); + return calculatedFieldDao.findByIdAsync(tenantId, calculatedFieldId.getId()); + } + @Override public List findAllCalculatedFields() { log.trace("Executing findAll"); @@ -95,10 +105,28 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements @Override public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); - calculatedFieldDao.removeById(tenantId, calculatedFieldId.getId()); + deleteEntity(tenantId, calculatedFieldId, false); + } + + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + CalculatedField calculatedField = calculatedFieldDao.findById(tenantId, id.getId()); + if (calculatedField == null) { + if (force) { + return; + } else { + throw new IncorrectParameterException("Unable to delete non-existent calculated field."); + } + } + deleteCalculatedField(tenantId, calculatedField); + } + + private void deleteCalculatedField(TenantId tenantId, CalculatedField calculatedField) { + log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedField.getId()); + calculatedFieldDao.removeById(tenantId, calculatedField.getUuidId()); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(calculatedField.getId()).entity(calculatedField).build()); } @Override @@ -125,6 +153,14 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findById(tenantId, calculatedFieldLinkId.getId()); } + @Override + public ListenableFuture findCalculatedFieldLinkByIdAsync(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId) { + log.trace("Executing findCalculatedFieldLinkByIdAsync [{}]", calculatedFieldLinkId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldLinkId, id -> "Incorrect calculatedFieldLinkId " + id); + return calculatedFieldLinkDao.findByIdAsync(tenantId, calculatedFieldLinkId.getId()); + } + @Override public List findAllCalculatedFieldLinks() { log.trace("Executing findAllCalculatedFieldLinks"); @@ -137,6 +173,12 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId); } + @Override + public ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findAllCalculatedFieldLinksByIdAsync, calculatedFieldId [{}]", calculatedFieldId); + return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldIdAsync(tenantId, calculatedFieldId); + } + @Override public PageData findAllCalculatedFieldLinks(PageLink pageLink) { log.trace("Executing findAllCalculatedFieldLinks, pageLink [{}]", pageLink); @@ -144,11 +186,6 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findAll(pageLink); } - @Override - public boolean existsByEntityId(TenantId tenantId, EntityId entityId) { - return calculatedFieldDao.existsByTenantIdAndEntityId(tenantId, entityId); - } - @Override public boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId) { return calculatedFieldDao.findAllByTenantId(tenantId).stream() diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index 4abe02a09b..d1c9fd86bc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -26,8 +26,6 @@ import java.util.List; public interface CalculatedFieldDao extends Dao { - boolean existsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId); - List findAllByTenantId(TenantId tenantId); List findAll(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index 728e19b890..34f2129bd7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.cf; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; @@ -28,6 +29,8 @@ public interface CalculatedFieldLinkDao extends Dao { List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); + ListenableFuture> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findAll(); PageData findAll(PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 1137b91947..bdc701070d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -18,7 +18,6 @@ package org.thingsboard.server.dao.sql.cf; import jakarta.transaction.Transactional; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; @@ -31,7 +30,6 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; -import org.thingsboard.server.dao.sql.device.NativeDeviceRepository; import org.thingsboard.server.dao.util.SqlDao; import java.util.List; @@ -46,11 +44,6 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId) { return DaoUtil.convertDataList(calculatedFieldRepository.findAllByTenantId(tenantId.getId())); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index a2f8f224c1..417b529dc9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.sql.cf; +import com.google.common.util.concurrent.ListenableFuture; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; @@ -22,7 +23,6 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -49,6 +49,11 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + return service.submit(() -> findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId)); + } + @Override public List findAll() { return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAll()); From 24281b48b5f3b720a5b963322c474a9346a97f77 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 18 Nov 2024 17:11:09 +0200 Subject: [PATCH 13/13] removed properrty from thingsboard.yml --- application/src/main/resources/thingsboard.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 699d2012cc..97405f35d3 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -424,10 +424,6 @@ sql: pool_size: "${SQL_RELATIONS_POOL_SIZE:4}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls query_timeout: "${SQL_RELATIONS_QUERY_TIMEOUT_SEC:20}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls -# Calculated Field parameters -calculatedField: - initFetchPackSize: "${INIT_FETCH_PACK_SIZE:50000}" - rocksdb: # Rocksdb path db_path: "${ROCKS_DB_PATH:${java.io.tmpdir}/rocksdb}"